iseplivescrap

2022-04-23 20:44:21 +02:00
parent 2648b0ac6d
commit 0395920ee5
1 changed files with 105 additions and 0 deletions
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import json
+import os
+import shelve
+import shutil
+from pathlib import Path
+
+import requests
+
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(override=True, verbose=True)
+    print(".env loaded !")
+except:
+    pass
+
+
+class Main:
+    def __init__(self, username, password):
+        self.username = username
+        self.password = password
+        self.headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0',
+            'Accept': 'application/json, text/plain, */*',
+            'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Content-Type': 'application/json;charset=utf-8',
+            'Origin': 'https://iseplive.fr',
+            'Connection': 'keep-alive',
+            'Referer': 'https://iseplive.fr/connexion',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+            'DNT': '1',
+            'Sec-GPC': '1',
+            'TE': 'trailers'
+        }
+
+        self.pathImage = Path("M:/Mon Drive/ISEPLIVE")
+        self.pathImage.mkdir(parents=True, exist_ok=True)
+        self.s = None
+        self.outputShelve = Path("./db")
+        with shelve.open(str(self.outputShelve.absolute())) as db:
+            self.annuaire: dict = db.get("annuaire", {})
+
+    def save(self):
+        with shelve.open(self.outputShelve) as db:
+            db["annuaire"]: dict = self.annuaire
+
+    def auth(self):
+        url = "https://iseplive.fr/api/auth"
+        payload = json.dumps({"username": self.username, "password": self.password})
+        response = requests.request("POST", url, headers=self.headers, data=payload)
+        data = json.loads(response.text)
+        self.token = data["token"]
+        self.refreshToken = data["refreshToken"]
+        self.authHeaders = {**self.headers, "Authorization": f"Bearer {self.token}",
+                            "X-Refresh-Token": self.refreshToken}
+
+        # self.s = requests.session(headers=self.authHeaders)
+        self.s = requests.session()
+        self.s.headers.update(self.authHeaders)
+
+    def saveStudent(self, dict_: dict):
+        print(dict_)
+        if (dict_["photoUrlThumb"] is None) or (not dict_["studentId"].isdigit()):
+            return False
+        r = self.s.get("https://iseplive.fr/api" + dict_["photoUrlThumb"], stream=True)
+        if r.status_code == 200:
+            with open(self.pathImage / f"{dict_['studentId']}.jpg", 'wb') as f:
+                r.raw.decode_content = True
+                shutil.copyfileobj(r.raw, f)
+        self.annuaire[int(dict_['studentId'])] = dict_
+
+    def scrapOnePage(self, filter_: tuple) -> bool:
+        # return True if it is last
+        response = self.s.get('https://iseplive.fr/api/user/student/search', params=filter_)
+        data = json.loads(response.text)
+        for dictStudent in data["content"]:
+            self.saveStudent(dictStudent)
+        return data["last"]
+
+    def scrapAll(self, param: tuple):
+        i = 0
+        tmp = param + (('page', str(i)),)
+        isLast = self.scrapOnePage(tmp)
+        while not isLast:
+            i += 1
+            tmp = param + (('page', str(i)),)
+            isLast = self.scrapOnePage(tmp)
+        print(f"Done scrap the param : {param}")
+
+
+if __name__ == '__main__':
+    a = Main(os.environ["USERNAME"], os.environ["PASSWORD"])
+    filter_ = (
+        ('name', ''),
+        ('promos', '2023'),
+        ('sort', 'a'),
+    )
+    a.auth()
+    a.scrapAll(filter_)
+    a.save()