Python：tweepy / psycopg2不将数据插入表中

Question

我通过建模this script将来自API的Twitter数据传输到Postgres数据库。使用这些确切的方法，我能够成功地将数据流式传输到两个表中（一个包含user_id / user_name，另一个包含数据）。我已经能够进行微小的改动以提取一些其他信息，但使用这些方法我只收集转发给定一个关键字列表，我想收集列表中的所有推文。根据原始脚本收集/存储转发user_ids和user_names的方式，我改变了尝试流式传输到新表的代码，而没有对转发进行任何引用。不幸的是，结果是两个空表。否则代码运行正常，并且正在向终端打印语句，没有数据。为什么会这样？以下是我的代码：

import psycopg2
import tweepy
import json
import numpy as np

# Importing postgres credentials
import postgres_credentials

# Importing twitter credentials
import twitter_credentials


# Accesing twitter from the App created in my account
def autorize_twitter_api():
"""
This function gets the consumer key, consumer secret key, access token
and access token secret given by the app created in your Twitter account
and authenticate them with Tweepy.
"""
# Get access and costumer key and tokens
auth = tweepy.OAuthHandler(twitter_credentials.CONSUMER_KEY, twitter_credentials.CONSUMER_SECRET)
auth.set_access_token(twitter_credentials.ACCESS_TOKEN, twitter_credentials.ACCESS_TOKEN_SECRET)

return auth


def create_tweets_table(term_to_search):
"""
This function open a connection with an already created database and creates a new table to
store tweets related to a subject specified by the user
"""

# Connect to Twitter Database created in Postgres
conn_twitter = psycopg2.connect(dbname=postgres_credentials.dbname, user=postgres_credentials.user, password=postgres_credentials.password, host=postgres_credentials.host,
                                    port=postgres_credentials.port)

# Create a cursor to perform database operations
cursor_twitter = conn_twitter.cursor()

# with the cursor now, create two tables, users twitter and the corresponding table according to the selected topic
cursor_twitter.execute("CREATE TABLE IF NOT EXISTS test_twitter_users (user_id VARCHAR PRIMARY KEY, user_name VARCHAR);")

query_create = "CREATE TABLE IF NOT EXISTS %s (id SERIAL, created_at_utc timestamp, tweet text NOT NULL, user_id VARCHAR, user_name VARCHAR, PRIMARY KEY(id), FOREIGN KEY(user_id) REFERENCES twitter_users(user_id));" % (
            "test_tweet_text")
cursor_twitter.execute(query_create)

# Commit changes
conn_twitter.commit()

# Close cursor and the connection
cursor_twitter.close()
conn_twitter.close()
return


def store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name):
"""
This function open a connection with an already created database and inserts into corresponding table
tweets related to the selected topic
"""

# Connect to Twitter Database created in Postgres
conn_twitter = psycopg2.connect(dbname=postgres_credentials.dbname, user=postgres_credentials.user, password=postgres_credentials.password, host=postgres_credentials.host,
                                    port=postgres_credentials.port)

# Create a cursor to perform database operations
cursor_twitter = conn_twitter.cursor()

# with the cursor now, insert tweet into table
cursor_twitter.execute(
    "INSERT INTO test_twitter_users (user_id, user_name) VALUES (%s, %s) ON CONFLICT(user_id) DO NOTHING;",
    (user_id, user_name))

cursor_twitter.execute(
    "INSERT INTO %s (created_at_utc, tweet, user_id, user_name) VALUES (%%s, %%s, %%s, %%s);" % (
                'test_tweet_text'),
    (created_at_utc, tweet, user_id, user_name))

# Commit changes
conn_twitter.commit()

# Close cursor and the connection
cursor_twitter.close()
conn_twitter.close()
return


class MyStreamListener(tweepy.StreamListener):
'''
def on_status(self, status):
    print(status.text)
'''

def on_data(self, raw_data):

    try:
        global term_to_search

        data = json.loads(raw_data)

        # Obtain all the variables to store in each column
        user_id = data['user']['id']
        user_name = data['user']['name']
        created_at_utc = data['created_at']
        tweet = data['text']

        # Store them in the corresponding table in the database
        store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name)

    except Exception as e:
        print(e)

def on_error(self, status_code):
    if status_code == 420:
        # returning False in on_error disconnects the stream
        return False

########################################################################

while True:
if __name__ == "__main__":
    # Creates the table for storing the tweets
    term_to_search = ["donald trump","trump"]
    create_tweets_table(term_to_search)

    # Connect to the streaming twitter API
    api = tweepy.API(wait_on_rate_limit_notify=True)

    # Stream the tweets
    try:
        streamer = tweepy.Stream(auth=autorize_twitter_api(), listener=MyStreamListener(api=api),tweet_mode='extended')
        streamer.filter(track=term_to_search)
    except:
        continue

Answer 1

如果在此函数中打印值会发生什么？你有价值吗？

def on_data(self, raw_data):

    try:
        global term_to_search

        data = json.loads(raw_data)

        # Obtain all the variables to store in each column
        user_id = data['user']['id']
        user_name = data['user']['name']
        created_at_utc = data['created_at']
        tweet = data['text']

        # Store them in the corresponding table in the database
        store_tweets_in_table(term_to_search, created_at_utc, tweet, user_id, user_name)

    except Exception as e:
        print(e)

当你打印sql语句时，你能看到没有数据的插入吗？

Answer 2

我发现了这个问题 - 我创建了两个新表，但是将数据插入到两个不同的表中。