#!/usr/bin/env python3

import json, urllib.request, time, os

CESIUMPLUS_URLS = [
	"https://g1.data.le-sou.org",
	"https://g1.data.mithril.re"
]
OUTFILE = "/var/www/html/g1/history/cesiumplusmap/archive_{}.json"
STATFILE = "/var/www/html/g1/history/cesiumplusmap/stat.csv"
GNUPLOT_FILE = "/var/www/html/g1/history/cesiumplusmap/stat.gnu"

if __name__ == "__main__":
	data = None
	for url in CESIUMPLUS_URLS:
		print("Request {}".format(url))
		for i in range(3):
			try:
				data = json.loads(urllib.request.urlopen(
					url+"/user/profile/_search",
					b'{"query":{"bool":{"must":[{"exists":{"field":"geoPoint"}}]}},"from":0,"size":'+str(min(10000, json.loads(urllib.request.urlopen(
						url+"/user/profile/_search",
						b'{"query":{"bool":{"must":[{"exists":{"field":"geoPoint"}}]}},"from":0,"size":0,"_source":[]}'
					).read().decode())["hits"]["total"])).encode()+b',"_source":["title","geoPoint","avatar._content_type"]}'
				).read().decode())["hits"]["hits"]
				break
			except Exception as e:
				print("Error: {} (retrying {}/3 in 10s)".format(e, i+1))
				time.sleep(10)
		if data:
			break
	if not data:
		print("Too many errors, quit")
		exit(1)
	
	print("Got {} results".format(len(data)))
	
	results = []
	for i in data:
		try:
			results.append([i["_id"],i["_source"].get("title", None),[i["_source"]["geoPoint"]["lat"],i["_source"]["geoPoint"]["lon"]]])
		except KeyError as e:
			print(i)
			print("KeyError:", e)
	
	tm = time.gmtime()
	tm = (str(tm.tm_year).zfill(4), str(tm.tm_mon).zfill(2), str(tm.tm_mday).zfill(2), str(tm.tm_hour).zfill(2), str(tm.tm_min).zfill(2),str(tm.tm_sec).zfill(2))
	
	f = open(OUTFILE.format("{}-{}-{}_{}-{}-{}".format(*tm)), "w")
	json.dump(results, f)
	f.close()
	
	f = open(STATFILE, "a")
	f.write("{}\t{}\n".format("{}{}{}{}{}{}".format(*tm), len(results)))
	f.close()
	
	os.system("gnuplot {}".format(GNUPLOT_FILE))