Deleded pycache and added timeout for sync.

This commit is contained in:
retoor 2024-11-25 20:27:39 +01:00
parent d6f15e3c46
commit f309353665
8 changed files with 16 additions and 8 deletions

View File

@ -40,21 +40,29 @@ async def get_recent_rants(start_from=1, page_size=10):
start_from += page_size
async def _sync_rants(start_from, page_size,count):
async for rant in get_recent_rants(start_from, page_size):
start_from += page_size
count += 1
rant["tags"] = json.dumps(rant["tags"])
db["rants"].upsert(rant, ["id"])
print(f"Upserted {count} rant(s).")
return count
async def sync_rants():
count = 0
start_from = 0
page_size = 20
try:
async for rant in get_recent_rants(start_from, page_size):
while True:
try:
count += await asyncio.wait_for(_sync_rants(start_from, page_size,count),5)
start_from += page_size
count += 1
rant["tags"] = json.dumps(rant["tags"])
db["rants"].upsert(rant, ["id"])
print(f"Upserted {count} rant(s).")
except:
print("Rate limit of server exceeded. That's normal.s")
except Exception as ex:
print(ex)
print("If exception described above is an timeout related error, it's due ratelimiting and considered OK.")
break
async def sync_comments():