diff --git a/tootbot.py b/tootbot.py index ddead15fad7de5976d198f423e2ee272fb74f34c..c41db23fb065db47a6a55f72a3d1411fb9c9bce8 100755 --- a/tootbot.py +++ b/tootbot.py @@ -12,6 +12,15 @@ import requests # Set to 1 to get some messages, 0 for error messages only debug=1 +# Posting delay (s), wait between mastodon posts, reduces the "burst" effect on timoeline, and instance workload if you hit rate limiters +posting_delay=1 + +# URL substrings dediated to source image hosting +twitter_pics_hosting="https://pbs.twitimg.com/" +nitter_pics_hosting="https://nitter.net/pic/" + +# Program logic below this line + if len(sys.argv) < 8: print("Usage: python3 tootbot.py twitter_account mastodon_login mastodon_passwd mastodon_instance max_days footer_tags delay mastodon_account_email") # noqa sys.exit(1) @@ -19,8 +28,7 @@ if len(sys.argv) < 8: # sqlite db to store processed tweets (and corresponding toots ids) sql = sqlite3.connect('tootbot.db') db = sql.cursor() -db.execute('''CREATE TABLE IF NOT EXISTS tweets (tweet text, toot text, - twitter text, mastodon text, instance text)''') +db.execute('''CREATE TABLE IF NOT EXISTS tweets (tweet text, toot text, twitter text, mastodon text, instance text)''') if len(sys.argv) > 4: instance = sys.argv[4] @@ -54,14 +62,14 @@ else: if debug: print("Processing", end='') -# Cut the source URL to avoid reposts +# Cut the source URL to avoid reposts when looking up the DB source=source.split("/search/")[0] -for t in reversed(d.entries): +for tweet in reversed(d.entries): # check if this tweet has been processed - db.execute('SELECT * FROM tweets WHERE tweet = ? AND twitter = ? and mastodon = ? and instance = ?', (t.id, source, mastodon, instance)) # noqa + db.execute('SELECT * FROM tweets WHERE tweet = ? AND twitter = ? and mastodon = ? and instance = ?', (tweet.id, source, mastodon, instance)) # noqa last = db.fetchone() - dt = t.published_parsed + dt = tweet.published_parsed age = datetime.now()-datetime(dt.tm_year, dt.tm_mon, dt.tm_mday, dt.tm_hour, dt.tm_min, dt.tm_sec) # process only unprocessed tweets less than 1 day old, after delay @@ -71,7 +79,7 @@ for t in reversed(d.entries): try: Mastodon.create_app('tootbot', api_base_url='https://'+instance, to_file='/var/run/lock/'+instance+'.secret') except: - print('ERROR: Failed to create app on instance '+instance) + print('FATAL ERROR: Failed to create app on instance '+instance) sys.exit(1) try: @@ -82,7 +90,7 @@ for t in reversed(d.entries): ) if debug: print(" ok.") except: - print("ERROR: Can't connect to Mastodon instance") + print("FATAL ERROR: Can't connect to Mastodon instance") sys.exit(1) if debug: print("Login with email ",mastodon_account_email," ...", end='') @@ -94,63 +102,74 @@ for t in reversed(d.entries): ) if debug: print(" ok.") except: - print("ERROR: First Login Failed !") + print("FATAL ERROR: First Login Failed !") sys.exit(1) - c = t.title + # construct the toot + toot_body = tweet.title if debug: print("============================================================") - if debug: print("[Posting]:",c) - if debug: print(" ") - if twitter and t.author.lower() != ('(@%s)' % twitter).lower(): - c = ("RT https://twitter.com/%s\n" % t.author[2:-1]) + c + if debug: print("[Posting]:",toot_body[0:60]) + if twitter and tweet.author.lower() != ('(@%s)' % twitter).lower(): + toot_body = ("RT https://twitter.com/%s\n" % tweet.author[2:-1]) + c toot_media = [] - # get the pictures... - for p in re.finditer(r"https://pbs.twimg.com/[^ \xa0\"]*", t.summary): - try: - media = requests.get(p.group(0)) - media_posted = mastodon_api.media_post(media.content, mime_type=media.headers.get('content-type')) - toot_media.append(media_posted['id']) - except: - print("ERROR: Can't get media !") + + # get the pictures... from twitter + if twitter_pics_hosting in tweet.summary: + for p in re.finditer(r"https://pbs.twimg.com/[^ \xa0\"]*", tweet.summary): + try: + media = requests.get(p.group(0)) + media_posted = mastodon_api.media_post(media.content, mime_type=media.headers.get('content-type')) + toot_media.append(media_posted['id']) + except: + print("ERROR: Can't get media !") + + # get the pictures... from nitter + if nitter_pics_hosting in tweet.summary: + for p in re.finditer(r"https://nitter.net/pic/[^ \xa0\"]*", tweet.summary): + try: + media = requests.get(p.group(0)) + media_posted = mastodon_api.media_post(media.content, mime_type=media.headers.get('content-type')) + toot_media.append(media_posted['id']) + except: + print("ERROR: Can't get media !") # replace short links by original URL - m = re.search(r"http[^ \xa0]*", c) + m = re.search(r"http[^ \xa0]*", toot_body) if m is not None: l = m.group(0) try: r = requests.get(l, allow_redirects=False) if r.status_code in {301, 302}: - c = c.replace(l, r.headers.get('Location')) + toot_body = toot_body.replace(l, r.headers.get('Location')) except: - print("ERROR: Can't get links !") + print("ERROR: Can't replace tweet's links !") # remove pic.twitter.com links - m = re.search(r"pic.twitter.com[^ \xa0]*", c) + m = re.search(r"pic.twitter.com[^ \xa0]*", toot_body) if m is not None: l = m.group(0) - c = c.replace(l, ' ') + toot_body = toot_body.replace(l, ' ') # remove ellipsis - c = c.replace('\xa0…', ' ') + toot_body = toot_body.replace('\xa0…', ' ') if twitter is None: - c = c + '\n\nSource: ' + t.authors[0].name + ' ' + t.link + toot_body = toot_body + '\n\nSource: ' + tweet.authors[0].name + ' ' + tweet.link if tags: if len(tags) > 2: - c = c + '\n' + tags + toot_body = toot_body + '\n' + tags if toot_media is not None: - toot = mastodon_api.status_post(c, in_reply_to_id=None, - media_ids=toot_media, - sensitive=False, - visibility='public', - spoiler_text=None) - if "id" in toot: - db.execute("INSERT INTO tweets VALUES ( ? , ? , ? , ? , ? )", - (t.id, toot["id"], source, mastodon, instance)) - sql.commit() - sleep(0.2) + try: + toot = mastodon_api.status_post(toot_body, in_reply_to_id=None, media_ids=toot_media, sensitive=False, visibility='public', spoiler_text=None) + if "id" in toot: + db.execute("INSERT INTO tweets VALUES ( ? , ? , ? , ? , ? )",(tweet.id, toot["id"], source, mastodon, instance)) + sql.commit() + except: + print("ERROR: Can't toot! Sorry, skipping this one") + if debug: print("Skipped toot:",c) + sleep(posting_delay) else: if debug: print(".",end='')