From d03b56f8dd1773aa98ffbeef10b4c79f20b0d353 Mon Sep 17 00:00:00 2001 From: Marc Partensky Date: Thu, 29 Oct 2020 12:42:02 +0100 Subject: [PATCH] added sklearn files --- Openclassrooms/Big-Data/WordCountMapper.py | 9 +++ Openclassrooms/Big-Data/WordCountReducer.py | 24 ++++++++ .../Big-Data/divide-and-distribute-to-rule.py | 23 ++++++++ Openclassrooms/Big-Data/hadoop.java | 58 +++++++++++++++++++ Openclassrooms/Big-Data/text.txt | 1 + Openclassrooms/Big-Data/wordcount.py | 15 +++++ Sklearn/test.py | 19 ++++++ Sklearn/test2.py | 20 +++++++ Sklearn/test3.py | 14 +++++ 9 files changed, 183 insertions(+) create mode 100755 Openclassrooms/Big-Data/WordCountMapper.py create mode 100755 Openclassrooms/Big-Data/WordCountReducer.py create mode 100755 Openclassrooms/Big-Data/divide-and-distribute-to-rule.py create mode 100644 Openclassrooms/Big-Data/hadoop.java create mode 100644 Openclassrooms/Big-Data/text.txt create mode 100755 Openclassrooms/Big-Data/wordcount.py create mode 100755 Sklearn/test.py create mode 100755 Sklearn/test2.py create mode 100755 Sklearn/test3.py diff --git a/Openclassrooms/Big-Data/WordCountMapper.py b/Openclassrooms/Big-Data/WordCountMapper.py new file mode 100755 index 0000000..6e7405c --- /dev/null +++ b/Openclassrooms/Big-Data/WordCountMapper.py @@ -0,0 +1,9 @@ +#! /usr/bin/env python + +import sys +for line in sys.stdin: + line = line.strip() + words = line.split() + + for word in words: + print("%s\t%d" % (word, 1)) diff --git a/Openclassrooms/Big-Data/WordCountReducer.py b/Openclassrooms/Big-Data/WordCountReducer.py new file mode 100755 index 0000000..7f85793 --- /dev/null +++ b/Openclassrooms/Big-Data/WordCountReducer.py @@ -0,0 +1,24 @@ +#! /usr/bin/env python + +import sys +total = 0 +lastword = None + +for line in sys.stdin: + line = line.strip() + + word, count = line.split() + count = int(count) + + if lastword is None: + lastword = word + + if word == lastword: + total += count + else: + print("%s\t%d occurrences" % (lastword, total)) + total = count + lastword = word + +if lastword is not None: + print("%s\t%d occurences" % (lastword, total)) diff --git a/Openclassrooms/Big-Data/divide-and-distribute-to-rule.py b/Openclassrooms/Big-Data/divide-and-distribute-to-rule.py new file mode 100755 index 0000000..18b37cc --- /dev/null +++ b/Openclassrooms/Big-Data/divide-and-distribute-to-rule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +from collections import defaultdict + + +text = "pour rendre le fonctionnement de mapreduce plus concret, nous allons \ +l'illustrer avec word count l'exemple typique de mapreduce, si typique qu'il en \ +est devenu le hello world de mapreduce et du calcul distribué" + +def word_count(text): + counts = defaultdict(int) + for word in text.split(): + counts[word.lower()] += 1 + return counts + +print(word_count(text)) + + +def map(key, value): + intermediate = [] + for i in value: + intermediate.append(i[1], (i[0], i[1:]))) + return intermediate diff --git a/Openclassrooms/Big-Data/hadoop.java b/Openclassrooms/Big-Data/hadoop.java new file mode 100644 index 0000000..1337218 --- /dev/null +++ b/Openclassrooms/Big-Data/hadoop.java @@ -0,0 +1,58 @@ +package ocr.dataArchitect.cours1.hadoop.exempleMapReduce; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +public class ExempleMapReduce extends Configured implements Tool { + + public int run(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("Usage: [input] [output]"); + System.exit(-1); + } + + // Création d'un job en lui fournissant la configuration et une description textuelle de la tâche + Job job = Job.getInstance(getConf()); + job.setJobName("notre probleme exemple"); + + // On précise les classes MyProgram, Map et Reduce + job.setJarByClass(ExempleMapReduce.class); + job.setMapperClass(ExampleMap.class); + job.setReducerClass(ExempleReduce.class); + + // Définition des types clé/valeur de notre problème + job.setMapOutputKeyClass(TypecleI.class); + job.setMapOutputValueClass(TypevalI.class); + + job.setOutputKeyClass(TypeCleS.class); + job.setOutputValueClass(TypeValS.class); + + // Définition des fichiers d'entrée et de sorties (ici considérés comme des arguments à préciser lors de l'exécution) + FileInputFormat.addInputPath(job, new Path(ourArgs[0])); + FileOutputFormat.setOutputPath(job, new Path(ourArgs[1])); + + //Suppression du fichier de sortie s'il existe déjà + FileSystem fs = FileSystem.newInstance(getConf()); + if (fs.exists(outputFilePath)) { + fs.delete(outputFilePath, true); + } + + return job.waitForCompletion(true) ? 0: 1; + } + + public static void main(String[] args) throws Exception { + ExempleMapReduce exempleDriver = new ExempleMapReduce(); + int res = ToolRunner.run(exempleDriver, args); + System.exit(res); + } +} diff --git a/Openclassrooms/Big-Data/text.txt b/Openclassrooms/Big-Data/text.txt new file mode 100644 index 0000000..0f281b0 --- /dev/null +++ b/Openclassrooms/Big-Data/text.txt @@ -0,0 +1 @@ +1 2 1 2 this is a test diff --git a/Openclassrooms/Big-Data/wordcount.py b/Openclassrooms/Big-Data/wordcount.py new file mode 100755 index 0000000..cbdbaa3 --- /dev/null +++ b/Openclassrooms/Big-Data/wordcount.py @@ -0,0 +1,15 @@ +#! /usr/bin/env python + +from pyspark import SparkContext + +sc = SparkContext() + +lines = sc.textFile("/Users/marcpartensky/Programs/Python/Machine-Learning/Openclassrooms/Big-Data/text.txt") + +word_counts = lines.flatMap(lambda line: line.split(' ')) \ + .map(lambda word: (word, 1)) \ + .reduceByKey(lambda count1, count2: count1+count2) \ + .collect() + +for (word, count) in word_counts: + print(word.encode("utf8"), count) diff --git a/Sklearn/test.py b/Sklearn/test.py new file mode 100755 index 0000000..172ba07 --- /dev/null +++ b/Sklearn/test.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.linear_model import PoissonRegressor +from sklearn.experimental import enable_hist_gradient_boosting # noqa +from sklearn.ensemble import HistGradientBoostingRegressor + +n_samples, n_features = 1000, 20 +rng = np.random.RandomState(0) +X = rng.randn(n_samples, n_features) +# positive integer target correlated with X[:, 5] with many zeros: +y = rng.poisson(lam=np.exp(X[:, 5]) / 2) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng) +glm = PoissonRegressor() +gbdt = HistGradientBoostingRegressor(loss='poisson', learning_rate=.01) +glm.fit(X_train, y_train) +gbdt.fit(X_train, y_train) +print(glm.score(X_test, y_test)) +print(gbdt.score(X_test, y_test)) diff --git a/Sklearn/test2.py b/Sklearn/test2.py new file mode 100755 index 0000000..f3d6e50 --- /dev/null +++ b/Sklearn/test2.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +from sklearn import set_config +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler +from sklearn.impute import SimpleImputer +from sklearn.compose import make_column_transformer +from sklearn.linear_model import LogisticRegression +set_config(display='diagram') + +num_proc = make_pipeline(SimpleImputer(strategy='median'), StandardScaler()) + +cat_proc = make_pipeline( + SimpleImputer(strategy='constant', fill_value='missing'), + OneHotEncoder(handle_unknown='ignore')) + +preprocessor = make_column_transformer((num_proc, ('feat1', 'feat3')), + (cat_proc, ('feat0', 'feat2'))) + +clf = make_pipeline(preprocessor, LogisticRegression()) +clf diff --git a/Sklearn/test3.py b/Sklearn/test3.py new file mode 100755 index 0000000..e7a0834 --- /dev/null +++ b/Sklearn/test3.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +import scipy +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.cluster import KMeans +from sklearn.datasets import make_blobs +from sklearn.metrics import completeness_score + +rng = np.random.RandomState(0) +X, y = make_blobs(random_state=rng) +X = scipy.sparse.csr_matrix(X) +X_train, X_test, _, y_test = train_test_split(X, y, random_state=rng) +kmeans = KMeans(algorithm='elkan').fit(X_train) +print(completeness_score(kmeans.predict(X_test), y_test))