Deleded pycache and added timeout for sync.
This commit is contained in:
parent
d6f15e3c46
commit
f309353665
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -40,21 +40,29 @@ async def get_recent_rants(start_from=1, page_size=10):
|
|||||||
start_from += page_size
|
start_from += page_size
|
||||||
|
|
||||||
|
|
||||||
async def sync_rants():
|
async def _sync_rants(start_from, page_size,count):
|
||||||
count = 0
|
|
||||||
start_from = 0
|
|
||||||
|
|
||||||
page_size = 20
|
|
||||||
|
|
||||||
try:
|
|
||||||
async for rant in get_recent_rants(start_from, page_size):
|
async for rant in get_recent_rants(start_from, page_size):
|
||||||
start_from += page_size
|
start_from += page_size
|
||||||
count += 1
|
count += 1
|
||||||
rant["tags"] = json.dumps(rant["tags"])
|
rant["tags"] = json.dumps(rant["tags"])
|
||||||
db["rants"].upsert(rant, ["id"])
|
db["rants"].upsert(rant, ["id"])
|
||||||
print(f"Upserted {count} rant(s).")
|
print(f"Upserted {count} rant(s).")
|
||||||
except:
|
return count
|
||||||
print("Rate limit of server exceeded. That's normal.s")
|
|
||||||
|
async def sync_rants():
|
||||||
|
count = 0
|
||||||
|
start_from = 0
|
||||||
|
|
||||||
|
page_size = 20
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
count += await asyncio.wait_for(_sync_rants(start_from, page_size,count),5)
|
||||||
|
start_from += page_size
|
||||||
|
except Exception as ex:
|
||||||
|
print(ex)
|
||||||
|
print("If exception described above is an timeout related error, it's due ratelimiting and considered OK.")
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
async def sync_comments():
|
async def sync_comments():
|
||||||
|
Loading…
Reference in New Issue
Block a user