106 lines
3.5 KiB
Python
106 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
|
|
import json
|
|
import os
|
|
import shelve
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
try:
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(override=True, verbose=True)
|
|
print(".env loaded !")
|
|
except:
|
|
pass
|
|
|
|
|
|
class Main:
|
|
def __init__(self, username, password):
|
|
self.username = username
|
|
self.password = password
|
|
self.headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0',
|
|
'Accept': 'application/json, text/plain, */*',
|
|
'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'Content-Type': 'application/json;charset=utf-8',
|
|
'Origin': 'https://iseplive.fr',
|
|
'Connection': 'keep-alive',
|
|
'Referer': 'https://iseplive.fr/connexion',
|
|
'Sec-Fetch-Dest': 'empty',
|
|
'Sec-Fetch-Mode': 'cors',
|
|
'Sec-Fetch-Site': 'same-origin',
|
|
'DNT': '1',
|
|
'Sec-GPC': '1',
|
|
'TE': 'trailers'
|
|
}
|
|
|
|
self.pathImage = Path("M:/Mon Drive/ISEPLIVE")
|
|
self.pathImage.mkdir(parents=True, exist_ok=True)
|
|
self.s = None
|
|
self.outputShelve = Path("./db")
|
|
with shelve.open(str(self.outputShelve.absolute())) as db:
|
|
self.annuaire: dict = db.get("annuaire", {})
|
|
|
|
def save(self):
|
|
with shelve.open(self.outputShelve) as db:
|
|
db["annuaire"]: dict = self.annuaire
|
|
|
|
def auth(self):
|
|
url = "https://iseplive.fr/api/auth"
|
|
payload = json.dumps({"username": self.username, "password": self.password})
|
|
response = requests.request("POST", url, headers=self.headers, data=payload)
|
|
data = json.loads(response.text)
|
|
self.token = data["token"]
|
|
self.refreshToken = data["refreshToken"]
|
|
self.authHeaders = {**self.headers, "Authorization": f"Bearer {self.token}",
|
|
"X-Refresh-Token": self.refreshToken}
|
|
|
|
# self.s = requests.session(headers=self.authHeaders)
|
|
self.s = requests.session()
|
|
self.s.headers.update(self.authHeaders)
|
|
|
|
def saveStudent(self, dict_: dict):
|
|
print(dict_)
|
|
if (dict_["photoUrlThumb"] is None) or (not dict_["studentId"].isdigit()):
|
|
return False
|
|
r = self.s.get("https://iseplive.fr/api" + dict_["photoUrlThumb"], stream=True)
|
|
if r.status_code == 200:
|
|
with open(self.pathImage / f"{dict_['studentId']}.jpg", 'wb') as f:
|
|
r.raw.decode_content = True
|
|
shutil.copyfileobj(r.raw, f)
|
|
self.annuaire[int(dict_['studentId'])] = dict_
|
|
|
|
def scrapOnePage(self, filter_: tuple) -> bool:
|
|
# return True if it is last
|
|
response = self.s.get('https://iseplive.fr/api/user/student/search', params=filter_)
|
|
data = json.loads(response.text)
|
|
for dictStudent in data["content"]:
|
|
self.saveStudent(dictStudent)
|
|
return data["last"]
|
|
|
|
def scrapAll(self, param: tuple):
|
|
i = 0
|
|
tmp = param + (('page', str(i)),)
|
|
isLast = self.scrapOnePage(tmp)
|
|
while not isLast:
|
|
i += 1
|
|
tmp = param + (('page', str(i)),)
|
|
isLast = self.scrapOnePage(tmp)
|
|
print(f"Done scrap the param : {param}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
a = Main(os.environ["USERNAME"], os.environ["PASSWORD"])
|
|
filter_ = (
|
|
('name', ''),
|
|
('promos', '2023'),
|
|
('sort', 'a'),
|
|
)
|
|
a.auth()
|
|
a.scrapAll(filter_)
|
|
a.save()
|