import tweepy
import json
import math
import glob
import csv
import zipfile
import zlib
from tweepy import TweepError
from time import sleep

from globals import POLITICIANS_INFO_FILE_PATH

try:
    with open(POLITICIANS_INFO_FILE_PATH) as data_file:
        users = json.load(data_file)
except:
    print("A problem occurred when parsing politicians_info.json")


with open('api_keys.json') as f:
    keys = json.load(f)

auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)

def store(counter):
    user = users[counter]
    userTweetHandle = user["twitter"][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)

    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)

    print('total ids: {}'.format(len(ids)))

    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)

    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)  # needed to prevent hitting API rate limit
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))

    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)


    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry["source"]:
            return entry["source"].split('>')[1].split('<')[0]
        else:
            return entry["source"]

    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {
                "created_at": entry["created_at"],
                "text": entry["text"],
                "in_reply_to_screen_name": entry["in_reply_to_screen_name"],
                "retweet_count": entry["retweet_count"],
                "favorite_count": entry["favorite_count"],
                "source": get_source(entry),
                "id_str": entry["id_str"],
                "is_retweet": is_retweet(entry)
            }
            results.append(t)

    print('creating minimized json master file')

    counter += 1
    
    if counter < len(users):
        store(counter)
    else:
        print('all done')


store(0)
8f5384abbbfccdb411a8d02b41423cea548d161b
<mask token>


def store(counter):
    user = users[counter]
    userTweetHandle = user['twitter'][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)
    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)
    print('total ids: {}'.format(len(ids)))
    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)
    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))
    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)
    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry['source']:
            return entry['source'].split('>')[1].split('<')[0]
        else:
            return entry['source']
    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {'created_at': entry['created_at'], 'text': entry['text'],
                'in_reply_to_screen_name': entry['in_reply_to_screen_name'],
                'retweet_count': entry['retweet_count'], 'favorite_count':
                entry['favorite_count'], 'source': get_source(entry),
                'id_str': entry['id_str'], 'is_retweet': is_retweet(entry)}
            results.append(t)
    print('creating minimized json master file')
    counter += 1
    if counter < len(users):
        store(counter)
    else:
        print('all done')


<mask token>

<mask token>
try:
    with open(POLITICIANS_INFO_FILE_PATH) as data_file:
        users = json.load(data_file)
except:
    print('A problem occurred when parsing politicians_info.json')
with open('api_keys.json') as f:
    keys = json.load(f)
<mask token>
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
<mask token>


def store(counter):
    user = users[counter]
    userTweetHandle = user['twitter'][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)
    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)
    print('total ids: {}'.format(len(ids)))
    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)
    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))
    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)
    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry['source']:
            return entry['source'].split('>')[1].split('<')[0]
        else:
            return entry['source']
    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {'created_at': entry['created_at'], 'text': entry['text'],
                'in_reply_to_screen_name': entry['in_reply_to_screen_name'],
                'retweet_count': entry['retweet_count'], 'favorite_count':
                entry['favorite_count'], 'source': get_source(entry),
                'id_str': entry['id_str'], 'is_retweet': is_retweet(entry)}
            results.append(t)
    print('creating minimized json master file')
    counter += 1
    if counter < len(users):
        store(counter)
    else:
        print('all done')


store(0)

<mask token>
try:
    with open(POLITICIANS_INFO_FILE_PATH) as data_file:
        users = json.load(data_file)
except:
    print('A problem occurred when parsing politicians_info.json')
with open('api_keys.json') as f:
    keys = json.load(f)
auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)


def store(counter):
    user = users[counter]
    userTweetHandle = user['twitter'][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)
    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)
    print('total ids: {}'.format(len(ids)))
    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)
    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))
    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)
    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry['source']:
            return entry['source'].split('>')[1].split('<')[0]
        else:
            return entry['source']
    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {'created_at': entry['created_at'], 'text': entry['text'],
                'in_reply_to_screen_name': entry['in_reply_to_screen_name'],
                'retweet_count': entry['retweet_count'], 'favorite_count':
                entry['favorite_count'], 'source': get_source(entry),
                'id_str': entry['id_str'], 'is_retweet': is_retweet(entry)}
            results.append(t)
    print('creating minimized json master file')
    counter += 1
    if counter < len(users):
        store(counter)
    else:
        print('all done')


store(0)

import tweepy
import json
import math
import glob
import csv
import zipfile
import zlib
from tweepy import TweepError
from time import sleep
from globals import POLITICIANS_INFO_FILE_PATH
try:
    with open(POLITICIANS_INFO_FILE_PATH) as data_file:
        users = json.load(data_file)
except:
    print('A problem occurred when parsing politicians_info.json')
with open('api_keys.json') as f:
    keys = json.load(f)
auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)


def store(counter):
    user = users[counter]
    userTweetHandle = user['twitter'][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)
    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)
    print('total ids: {}'.format(len(ids)))
    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)
    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))
    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)
    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry['source']:
            return entry['source'].split('>')[1].split('<')[0]
        else:
            return entry['source']
    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {'created_at': entry['created_at'], 'text': entry['text'],
                'in_reply_to_screen_name': entry['in_reply_to_screen_name'],
                'retweet_count': entry['retweet_count'], 'favorite_count':
                entry['favorite_count'], 'source': get_source(entry),
                'id_str': entry['id_str'], 'is_retweet': is_retweet(entry)}
            results.append(t)
    print('creating minimized json master file')
    counter += 1
    if counter < len(users):
        store(counter)
    else:
        print('all done')


store(0)

import tweepy
import json
import math
import glob
import csv
import zipfile
import zlib
from tweepy import TweepError
from time import sleep

from globals import POLITICIANS_INFO_FILE_PATH

try:
    with open(POLITICIANS_INFO_FILE_PATH) as data_file:
        users = json.load(data_file)
except:
    print("A problem occurred when parsing politicians_info.json")


with open('api_keys.json') as f:
    keys = json.load(f)

auth = tweepy.OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = tweepy.API(auth)

def store(counter):
    user = users[counter]
    userTweetHandle = user["twitter"][1:]
    output_file = './output/tweets/{}_tweets.json'.format(userTweetHandle)

    with open('./output/{}.json'.format(userTweetHandle)) as f:
        ids = json.load(f)

    print('total ids: {}'.format(len(ids)))

    all_data = []
    start = 0
    end = 200
    limit = len(ids)
    i = math.ceil(limit / 200)

    for go in range(i):
        print('currently getting {} - {}'.format(start, end))
        sleep(6)  # needed to prevent hitting API rate limit
        id_batch = ids[start:end]
        start += 200
        end += 200
        tweets = api.statuses_lookup(id_batch)
        for tweet in tweets:
            all_data.append(dict(tweet._json))

    print('metadata collection complete')
    print('creating master json file')
    with open(output_file, 'w') as outfile:
        json.dump(all_data, outfile)


    results = []

    def is_retweet(entry):
        return 'retweeted_status' in entry.keys()

    def get_source(entry):
        if '<' in entry["source"]:
            return entry["source"].split('>')[1].split('<')[0]
        else:
            return entry["source"]

    with open(output_file) as json_data:
        data = json.load(json_data)
        for entry in data:
            t = {
                "created_at": entry["created_at"],
                "text": entry["text"],
                "in_reply_to_screen_name": entry["in_reply_to_screen_name"],
                "retweet_count": entry["retweet_count"],
                "favorite_count": entry["favorite_count"],
                "source": get_source(entry),
                "id_str": entry["id_str"],
                "is_retweet": is_retweet(entry)
            }
            results.append(t)

    print('creating minimized json master file')

    counter += 1
    
    if counter < len(users):
        store(counter)
    else:
        print('all done')


store(0)