diff --git a/iseplivescrap.py b/iseplivescrap.py new file mode 100644 index 0000000..49c0ac7 --- /dev/null +++ b/iseplivescrap.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python + +import json +import os +import shelve +import shutil +from pathlib import Path + +import requests + +try: + from dotenv import load_dotenv + + load_dotenv(override=True, verbose=True) + print(".env loaded !") +except: + pass + + +class Main: + def __init__(self, username, password): + self.username = username + self.password = password + self.headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3', + 'Accept-Encoding': 'gzip, deflate, br', + 'Content-Type': 'application/json;charset=utf-8', + 'Origin': 'https://iseplive.fr', + 'Connection': 'keep-alive', + 'Referer': 'https://iseplive.fr/connexion', + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin', + 'DNT': '1', + 'Sec-GPC': '1', + 'TE': 'trailers' + } + + self.pathImage = Path("M:/Mon Drive/ISEPLIVE") + self.pathImage.mkdir(parents=True, exist_ok=True) + self.s = None + self.outputShelve = Path("./db") + with shelve.open(str(self.outputShelve.absolute())) as db: + self.annuaire: dict = db.get("annuaire", {}) + + def save(self): + with shelve.open(self.outputShelve) as db: + db["annuaire"]: dict = self.annuaire + + def auth(self): + url = "https://iseplive.fr/api/auth" + payload = json.dumps({"username": self.username, "password": self.password}) + response = requests.request("POST", url, headers=self.headers, data=payload) + data = json.loads(response.text) + self.token = data["token"] + self.refreshToken = data["refreshToken"] + self.authHeaders = {**self.headers, "Authorization": f"Bearer {self.token}", + "X-Refresh-Token": self.refreshToken} + + # self.s = requests.session(headers=self.authHeaders) + self.s = requests.session() + self.s.headers.update(self.authHeaders) + + def saveStudent(self, dict_: dict): + print(dict_) + if (dict_["photoUrlThumb"] is None) or (not dict_["studentId"].isdigit()): + return False + r = self.s.get("https://iseplive.fr/api" + dict_["photoUrlThumb"], stream=True) + if r.status_code == 200: + with open(self.pathImage / f"{dict_['studentId']}.jpg", 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + self.annuaire[int(dict_['studentId'])] = dict_ + + def scrapOnePage(self, filter_: tuple) -> bool: + # return True if it is last + response = self.s.get('https://iseplive.fr/api/user/student/search', params=filter_) + data = json.loads(response.text) + for dictStudent in data["content"]: + self.saveStudent(dictStudent) + return data["last"] + + def scrapAll(self, param: tuple): + i = 0 + tmp = param + (('page', str(i)),) + isLast = self.scrapOnePage(tmp) + while not isLast: + i += 1 + tmp = param + (('page', str(i)),) + isLast = self.scrapOnePage(tmp) + print(f"Done scrap the param : {param}") + + +if __name__ == '__main__': + a = Main(os.environ["USERNAME"], os.environ["PASSWORD"]) + filter_ = ( + ('name', ''), + ('promos', '2023'), + ('sort', 'a'), + ) + a.auth() + a.scrapAll(filter_) + a.save()