from sip import setapi
setapi("QDate", 2)
setapi("QDateTime", 2)
setapi("QTextStream", 2)
setapi("QTime", 2)
setapi("QVariant", 2)
setapi("QString", 2)
setapi("QUrl", 2)
import os
import json
import sys

reload(sys)
sys.setdefaultencoding('utf-8')


#change the directory
directory = '/home/shubhi/IR/IRdata1'
attribute_list = ["tweet_text", "user_name", "verified_user", "followers", "retweet_count", "hashtags", "tweet_id"]
count = 0
for filename in os.listdir(directory):
	filepath = directory + '/' + str(filename)
	#print(str(filepath))
	f = open(filepath, "r")
	data = f.read()
	delim = "}"
	data = data.split(delim)
	for tweet in data:
		#tweet_count = tweet_count + 1
		dict_item = str(tweet) + delim
		
		#count = count + 1
		try:
			json_item = json.loads(dict_item)
		except Exception as e1:
			#print e1
			#print dict_item
			#try_count = try_count+1
			#print("json error")
			#json_error+=1
			index = dict_item.find("description")
			dict_item = dict_item[0:index] + ',' + dict_item[index:]
			try:
				json_item = json.loads(dict_item)
			except:
				#print "another json_error"
				#print dict_item
				continue
		#write_count = write_count+1
		try:
			count+=1
			'''
			for attr in attribute_list:
				#print("for")
				attrfile = attr + ".txt"
				f = open(attrfile, "a+")
				#print(attr, " ", json_item[attr])
				
				f.write(str(json_item[attr]) + "\n")
			'''
			attrfile1 = 'hashtags' + ".txt"
			attrfile2 = 'tweet_text' + ".txt"
			f1 = open(attrfile1, "a+")
			f2 = open(attrfile2, "a+")
			tweet_text = str(json_item["tweet_text"]).strip('\n')
			tweet_text = tweet_text.replace("\n", " ")
			f2.write( tweet_text+ "\n")
			f1.write( str(json_item["hashtags"])+ "\n")
		except Exception as e:
			print(e)
			print("writeerror")
	f.close()
print(count)
16d089b31bd2928526449d63541abfea2da16c31
<mask token>

<mask token>
setapi('QDate', 2)
setapi('QDateTime', 2)
setapi('QTextStream', 2)
setapi('QTime', 2)
setapi('QVariant', 2)
setapi('QString', 2)
setapi('QUrl', 2)
<mask token>
reload(sys)
sys.setdefaultencoding('utf-8')
<mask token>
for filename in os.listdir(directory):
    filepath = directory + '/' + str(filename)
    f = open(filepath, 'r')
    data = f.read()
    delim = '}'
    data = data.split(delim)
    for tweet in data:
        dict_item = str(tweet) + delim
        try:
            json_item = json.loads(dict_item)
        except Exception as e1:
            index = dict_item.find('description')
            dict_item = dict_item[0:index] + ',' + dict_item[index:]
            try:
                json_item = json.loads(dict_item)
            except:
                continue
        try:
            count += 1
            """
			for attr in attribute_list:
				#print("for")
				attrfile = attr + ".txt"
				f = open(attrfile, "a+")
				#print(attr, " ", json_item[attr])
				
				f.write(str(json_item[attr]) + "
")
			"""
            attrfile1 = 'hashtags' + '.txt'
            attrfile2 = 'tweet_text' + '.txt'
            f1 = open(attrfile1, 'a+')
            f2 = open(attrfile2, 'a+')
            tweet_text = str(json_item['tweet_text']).strip('\n')
            tweet_text = tweet_text.replace('\n', ' ')
            f2.write(tweet_text + '\n')
            f1.write(str(json_item['hashtags']) + '\n')
        except Exception as e:
            print(e)
            print('writeerror')
    f.close()
print(count)

<mask token>
setapi('QDate', 2)
setapi('QDateTime', 2)
setapi('QTextStream', 2)
setapi('QTime', 2)
setapi('QVariant', 2)
setapi('QString', 2)
setapi('QUrl', 2)
<mask token>
reload(sys)
sys.setdefaultencoding('utf-8')
directory = '/home/shubhi/IR/IRdata1'
attribute_list = ['tweet_text', 'user_name', 'verified_user', 'followers',
    'retweet_count', 'hashtags', 'tweet_id']
count = 0
for filename in os.listdir(directory):
    filepath = directory + '/' + str(filename)
    f = open(filepath, 'r')
    data = f.read()
    delim = '}'
    data = data.split(delim)
    for tweet in data:
        dict_item = str(tweet) + delim
        try:
            json_item = json.loads(dict_item)
        except Exception as e1:
            index = dict_item.find('description')
            dict_item = dict_item[0:index] + ',' + dict_item[index:]
            try:
                json_item = json.loads(dict_item)
            except:
                continue
        try:
            count += 1
            """
			for attr in attribute_list:
				#print("for")
				attrfile = attr + ".txt"
				f = open(attrfile, "a+")
				#print(attr, " ", json_item[attr])
				
				f.write(str(json_item[attr]) + "
")
			"""
            attrfile1 = 'hashtags' + '.txt'
            attrfile2 = 'tweet_text' + '.txt'
            f1 = open(attrfile1, 'a+')
            f2 = open(attrfile2, 'a+')
            tweet_text = str(json_item['tweet_text']).strip('\n')
            tweet_text = tweet_text.replace('\n', ' ')
            f2.write(tweet_text + '\n')
            f1.write(str(json_item['hashtags']) + '\n')
        except Exception as e:
            print(e)
            print('writeerror')
    f.close()
print(count)

from sip import setapi
setapi('QDate', 2)
setapi('QDateTime', 2)
setapi('QTextStream', 2)
setapi('QTime', 2)
setapi('QVariant', 2)
setapi('QString', 2)
setapi('QUrl', 2)
import os
import json
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
directory = '/home/shubhi/IR/IRdata1'
attribute_list = ['tweet_text', 'user_name', 'verified_user', 'followers',
    'retweet_count', 'hashtags', 'tweet_id']
count = 0
for filename in os.listdir(directory):
    filepath = directory + '/' + str(filename)
    f = open(filepath, 'r')
    data = f.read()
    delim = '}'
    data = data.split(delim)
    for tweet in data:
        dict_item = str(tweet) + delim
        try:
            json_item = json.loads(dict_item)
        except Exception as e1:
            index = dict_item.find('description')
            dict_item = dict_item[0:index] + ',' + dict_item[index:]
            try:
                json_item = json.loads(dict_item)
            except:
                continue
        try:
            count += 1
            """
			for attr in attribute_list:
				#print("for")
				attrfile = attr + ".txt"
				f = open(attrfile, "a+")
				#print(attr, " ", json_item[attr])
				
				f.write(str(json_item[attr]) + "
")
			"""
            attrfile1 = 'hashtags' + '.txt'
            attrfile2 = 'tweet_text' + '.txt'
            f1 = open(attrfile1, 'a+')
            f2 = open(attrfile2, 'a+')
            tweet_text = str(json_item['tweet_text']).strip('\n')
            tweet_text = tweet_text.replace('\n', ' ')
            f2.write(tweet_text + '\n')
            f1.write(str(json_item['hashtags']) + '\n')
        except Exception as e:
            print(e)
            print('writeerror')
    f.close()
print(count)

from sip import setapi
setapi("QDate", 2)
setapi("QDateTime", 2)
setapi("QTextStream", 2)
setapi("QTime", 2)
setapi("QVariant", 2)
setapi("QString", 2)
setapi("QUrl", 2)
import os
import json
import sys

reload(sys)
sys.setdefaultencoding('utf-8')


#change the directory
directory = '/home/shubhi/IR/IRdata1'
attribute_list = ["tweet_text", "user_name", "verified_user", "followers", "retweet_count", "hashtags", "tweet_id"]
count = 0
for filename in os.listdir(directory):
	filepath = directory + '/' + str(filename)
	#print(str(filepath))
	f = open(filepath, "r")
	data = f.read()
	delim = "}"
	data = data.split(delim)
	for tweet in data:
		#tweet_count = tweet_count + 1
		dict_item = str(tweet) + delim
		
		#count = count + 1
		try:
			json_item = json.loads(dict_item)
		except Exception as e1:
			#print e1
			#print dict_item
			#try_count = try_count+1
			#print("json error")
			#json_error+=1
			index = dict_item.find("description")
			dict_item = dict_item[0:index] + ',' + dict_item[index:]
			try:
				json_item = json.loads(dict_item)
			except:
				#print "another json_error"
				#print dict_item
				continue
		#write_count = write_count+1
		try:
			count+=1
			'''
			for attr in attribute_list:
				#print("for")
				attrfile = attr + ".txt"
				f = open(attrfile, "a+")
				#print(attr, " ", json_item[attr])
				
				f.write(str(json_item[attr]) + "\n")
			'''
			attrfile1 = 'hashtags' + ".txt"
			attrfile2 = 'tweet_text' + ".txt"
			f1 = open(attrfile1, "a+")
			f2 = open(attrfile2, "a+")
			tweet_text = str(json_item["tweet_text"]).strip('\n')
			tweet_text = tweet_text.replace("\n", " ")
			f2.write( tweet_text+ "\n")
			f1.write( str(json_item["hashtags"])+ "\n")
		except Exception as e:
			print(e)
			print("writeerror")
	f.close()
print(count)