[PYTHON] Get Twitter userData

Since there is only screen_name in the follower list of twitter, I decided to add the user id information to the follower list of twitter.

Twitter API GET users/show https://dev.twitter.com/rest/reference/get/users/show

Twitter follower list data (csv)

API limit users / show up to 900/15 minutes

API restrictions are troublesome on Twitter, but the way to avoid it is by the method of the administrator of Code 7 Ward. I am always grateful for your help. Get a lot of tweets with TwitterAPI. Consider server-side errors (in python)

twitter_user_get.py

# coding=utf-8

import sys
import tweetkey
import os
from requests_oauthlib import OAuth1Session
import csv
import time
import json
import datetime


def init():

    #Set various keys from the usual external file
    CONSUMER_KEY = tweetkey.twkey['CONSUMER_KEY']
    CONSUMER_SECRET = tweetkey.twkey['CONSUMER_SECRET']
    ACCESS_TOKEN = tweetkey.twkey['ACCESS_TOKEN']
    ACCESS_SECRET = tweetkey.twkey['ACCESS_SECRET']

    twitter = OAuth1Session(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET)

    return(twitter)

def twittergetter(input_name,output_name,api):

    TweetIDList = []
    with open(input_name, 'r') as f:
        reader = csv.reader(f)
        header = next(reader)  #When you want to skip the header

        #Added ID column to HEAD for output
        header.append("id")

        fout = open(output_name, 'wt')
        writer = csv.writer(fout)
        writer.writerow(header)

        for row in reader:
            #Get account from array 2 of csv and throw
            screen_name = row[2]
            print(screen_name)
            url = "https://api.twitter.com/1.1/users/show.json?screen_name=" + screen_name
            # params = {'count': 100}
            # get_users has no parameters
            req = api.get(url)

            if req.status_code == 200:
                users = json.loads(req.text)
                print(users)
                #userid is in id
                id = users["id"]
                print(id)
                #Add the returned id to the array and write
                row.append(id)
                writer.writerow(row)

            else:
　　　　　　　　　　#At 404, screen_Since the name has already changed or withdrawal, write as it is
                if req.status_code == 404:
                    writer.writerow(row)
                else:
                    print ("Error: %d" % req.status_code)
                    time.sleep(240)

            #Header confirmation (number of times limit)
            # X-Rate-Limit-Check because it is rare that Remaining is not included
            if ('X-Rate-Limit-Remaining' in req.headers and 'X-Rate-Limit-Reset' in req.headers):
                if (int(req.headers['X-Rate-Limit-Remaining']) == 0):
                    waitUntilReset(int(req.headers['X-Rate-Limit-Reset']))
                    checkLimit()
            else:
                print('not found  -  X-Rate-Limit-Remaining or X-Rate-Limit-Reset')
                checkLimit()

        fout.close()
    return()


def checkLimit():
    '''
Query the limit and wait until it becomes accessible
    '''
    unavailableCnt = 0
    while True:
        url = "https://api.twitter.com/1.1/application/rate_limit_status.json"
        res = api.get(url)
        print(res.text)

        if res.status_code == 503:
            # 503 : Service Unavailable
            if unavailableCnt > 10:
                raise Exception('Twitter API error %d' % res.status_code)

            unavailableCnt += 1
            print('Service Unavailable 503')
            waitUntilReset(time.mktime(datetime.datetime.now().timetuple()) + 30)
            continue

        unavailableCnt = 0

        if res.status_code != 200:
            raise Exception('Twitter API error %d' % res.status_code)

        remaining, reset = getLimitContext(json.loads(res.text))
        if (remaining == 0):
            waitUntilReset(reset)
        else:
            break

def getLimitContext(res_text):
    '''
Get information on the number of times limit (at startup)
    '''
    remaining = res_text['resources']['users']['/users/show/:id']['remaining']
    reset = res_text['resources']['users']['/users/show/:id']['reset']
    print(reset)
    return int(remaining), int(reset)

def waitUntilReset(reset):
    '''
sleep until reset time
    '''
    seconds = reset - time.mktime(datetime.datetime.now().timetuple())
    seconds = max(seconds, 0)
    print('\n     =====================')
    print('     == waiting %d sec ==' % seconds)
    print('     =====================')
    sys.stdout.flush()
    time.sleep(seconds + 10)  #Just in case+10 seconds

if __name__ == '__main__':
    api = init()
    input_name = os.getcwd() + "/twitterList.csv"
    output_name = os.getcwd() + "/twitterList_add_id.csv"
    twittergetter(input_name,output_name, api)

However, when I try to convert screen_name for 120,000 people, it doesn't end even after 2 days. .. ..