user info, replies, and user timeline inline with twint and bugfix fo…

…r "created at" datetime bug

modules/Neo4jDataAccess.py

@@ -473,10 +473,6 @@ def __enrich_usr_info(self, df):
         return dfs
 
     def save_twintdf_to_neo(self, df, job_name, job_id=None):
-        if (df is None) or (len(df) == 0):
-            logger.info('Empty df for neo conversion, skip')
-            return None
-
         twintdftic = time.perf_counter()
         df = TwintPool().twint_df_to_neo4j_df(df)
         twintdftoc = time.perf_counter()
@@ -506,6 +502,7 @@ def save_twintdf_to_neo(self, df, job_name, job_id=None):
                 params.append(pd.DataFrame([{'tweet_id': int(row['status_id']),
                                              'text': row['full_text'],
                                              'created_at': str(pd.to_datetime(row['created_at'])),
+                                             'date': str(pd.to_datetime(row['date'])),
                                              'favorite_count': row['favorite_count'],
                                              'retweet_count': row['retweet_count'],
                                              'type': tweet_type,
@@ -590,21 +587,21 @@ def __parse_urls_twint(self, df, job_name, job_id):
             for index, row in df.iterrows():
                 if row["urls"]:
                     urls = [url for url in row["urls"]]
-                    parsed = URL(urls[counter])
+                    parsed = urlparse(urls[counter])
                     url_params_lst.append(pd.DataFrame([{
                         'tweet_id': int(row["status_id"]),
                         'full_url': urls[counter],
                         'job_id': job_id,
                         'job_name': job_name,
                         'schema': parsed.scheme,
-                        'netloc': parsed.authority,
+                        'netloc': parsed.netloc,
                         'path': parsed.path,
-                        'params': '',#parsed.params,
+                        'params': parsed.params,
                         'query': parsed.query,
                         'fragment': parsed.fragment,
                         'username': parsed.username,
-                        'password': parsed.authorization,
-                        'hostname': parsed.host,
+                        'password': parsed.password,
+                        'hostname': parsed.hostname,
                         'port': parsed.port}]))
         except Exception as e:
             logging.error('params.append exn', e)
@@ -667,9 +664,9 @@ def __tweetdf_to_neo_account_df(self, df, job_name):
         acctdf['screen_name'] = df['username']
         acctdf['friends_count'] = df["following"]
         acctdf['followers_count'] = df["followers"]
+        acctdf['created_at'] = df['created_at']
         acctdf['job_name'] = str(job_name)
         return acctdf
-
     def write_twint_enriched_tweetdf_to_neo(self, res, job_name, job_id):
         graph = self.__get_neo4j_graph('writer')
         global_tic = time.perf_counter()

modules/TwintPool.py

@@ -71,33 +71,40 @@ def _get_term(self, Search="IngSoc", Since="1984-04-20 13:00:00", Until="1984-04
         toc = time.perf_counter()
         logger.info(f'finished searching for tweets in:  {toc - tic:0.4f} seconds')
 
-    def _get_timeline(self, username, limit):
+    def _get_Replies_to_user(self,username, limit):
         self.config.Retweets = True
-        self.config.Search = "from:" + username
+        self.config.Search =  username
         self.config.Limit = limit
+        self.config.Pandas = True
+        self.config.Hide_output = True
         twint.run.Search(self.config)
-        tweets_df = twint.storage.panda.Tweets_df
-        return tweets_df
+        return twint.storage.panda.Tweets_df
+
+    def _get_user_timeline(self,username,limit):
+        self.config.Profile_full = True
+        self.config.Retweets = True
+        self.config.Limit = limit
+        self.config.Username = username
+        self.config.Pandas = True
+        self.config.Hide_output = True
+        twint.run.Profile(self.config)
+        return twint.storage.panda.Tweets_df
 
-    def _get_user_info(self, username):
+    def _get_user_info(self,username):
         self.config.User_full = True
         self.config.Username = username
         self.config.Pandas = True
+        self.config.Hide_output=True
         twint.run.Lookup(self.config)
         return twint.storage.panda.User_df
 
     def check_hydrate(self, df):
 
         df = df.assign(id=df['id'].astype('int64'))
 
-        df2 = df.drop_duplicates(subset=['id'])
-        if len(df2) < len(df):
-            logger.warning('Deduplicating input tweet df had duplicates, %s -> %s', len(df), len(df2))
-        df = df2
-
         from .Neo4jDataAccess import Neo4jDataAccess
         neo4j_creds = None
-        with open('neo4jcreds.json') as json_file:
+        with open('/secrets/neo4jcreds.json') as json_file:
             neo4j_creds = json.load(json_file)
 
         # dft : df[[id:int64, hydrated: NaN | 'FULL' | 'PARTIAL'??]]
@@ -124,7 +131,7 @@ def twint_df_to_neo4j_df(self, df):
         def row_to_tweet_type(row):
             if row['quote_url'] is None or row['quote_url'] == '':
                 return "QUOTE_RETWEET"
-            elif ('retweet' in row) and row['retweet']:
+            elif row['retweet']:
                 return "RETWEET"
             elif row['id'] == row['conversation_id']:
                 return "TWEET"
@@ -140,21 +147,23 @@ def row_to_tweet_type(row):
         # return None
 
         def row_tweet_to_urls(row):
+            extractor = URLExtract()
             return list(extractor.gen_urls(row['tweet']))
 
         neo4j_df['user_location'] = None
-        neo4j_df['tweet_type_twint'] = df.apply(row_to_tweet_type, axis=1, result_type='reduce') #handle empty df
+        neo4j_df['tweet_type_twint'] = df.apply(row_to_tweet_type, axis=1)
         neo4j_df['hashtags'] = df['hashtags'].apply(lambda x: [{'text': ht} for ht in x])
         neo4j_df['user_followers_count'] = None
         neo4j_df['user_friends_count'] = None
+        neo4j_df['date'] = df['date']
         # neo4j_df['user_created_at'] = None
         neo4j_df['user_profile_image_url'] = None
         neo4j_df['reply_tweet_id'] = None
         neo4j_df['user_mentions'] = df['tweet'].str.findall('@[\w]+')
         # neo4j_df['retweet_id'] is suspiciously empty (always)
         neo4j_df['retweeted_status'] = None
         neo4j_df['conversation_id'] = df['conversation_id']  # FIXME no-op?
-        neo4j_df['created_at'] = (neo4j_df['created_at'] / 1000).apply(lambda n: datetime.fromtimestamp(n))
+        neo4j_df['created_at'] = (df['created_at'] / 1000).apply(lambda n: datetime.fromtimestamp(n))
 
         # neo4j_df['quoted_status_id'] = df.apply(row_to_quoted_status_id, axis=1)
         # neo4j_df['is_quote_status'] = neo4j_df['quoted_status_id'] != None
@@ -167,8 +176,3 @@ def row_tweet_to_urls(row):
 
     def to_arrow(self, tweets_df):
         pass
-
-
-
-
-