added sklearn files

Openclassrooms/Big-Data/WordCountMapper.py

@@ -0,0 +1,9 @@
+#! /usr/bin/env python
+
+import sys
+for line in sys.stdin:
+    line = line.strip()
+    words = line.split()
+
+    for word in words:
+        print("%s\t%d" % (word, 1))

Openclassrooms/Big-Data/WordCountReducer.py

@@ -0,0 +1,24 @@
+#! /usr/bin/env python
+
+import sys
+total = 0
+lastword = None
+
+for line in sys.stdin:
+    line = line.strip()
+
+    word, count = line.split()
+    count = int(count)
+
+    if lastword is None:
+        lastword = word
+
+    if word == lastword:
+        total += count
+    else:
+        print("%s\t%d occurrences" % (lastword, total))
+        total = count
+        lastword = word
+
+if lastword is not None:
+    print("%s\t%d occurences" % (lastword, total))

Openclassrooms/Big-Data/divide-and-distribute-to-rule.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from collections import defaultdict
+
+
+text = "pour rendre le fonctionnement de mapreduce plus concret, nous allons \
+l'illustrer avec word count l'exemple typique de mapreduce, si typique qu'il en \
+est devenu le hello world de mapreduce et du calcul distribué"
+
+def word_count(text):
+    counts = defaultdict(int)
+    for word in text.split():
+        counts[word.lower()] += 1
+    return counts
+
+print(word_count(text))
+
+
+def map(key, value):
+    intermediate = []
+    for i in value:
+        intermediate.append(i[1], (i[0], i[1:])))
+    return intermediate

Openclassrooms/Big-Data/hadoop.java

@@ -0,0 +1,58 @@
+package ocr.dataArchitect.cours1.hadoop.exempleMapReduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+public class ExempleMapReduce extends Configured implements Tool {
+
+    public int run(String[] args) throws Exception {
+        if (args.length != 2) {
+            System.out.println("Usage: [input] [output]");
+            System.exit(-1);
+        }
+
+        // Création d'un job en lui fournissant la configuration et une description textuelle de la tâche
+        Job job = Job.getInstance(getConf());
+        job.setJobName("notre probleme exemple");
+
+        // On précise les classes MyProgram, Map et Reduce
+        job.setJarByClass(ExempleMapReduce.class);
+        job.setMapperClass(ExampleMap.class);
+        job.setReducerClass(ExempleReduce.class);
+
+        // Définition des types clé/valeur de notre problème
+        job.setMapOutputKeyClass(TypecleI.class);
+        job.setMapOutputValueClass(TypevalI.class);
+
+        job.setOutputKeyClass(TypeCleS.class);
+        job.setOutputValueClass(TypeValS.class);
+
+        // Définition des fichiers d'entrée et de sorties (ici considérés comme des arguments à préciser lors de l'exécution)
+        FileInputFormat.addInputPath(job, new Path(ourArgs[0]));
+        FileOutputFormat.setOutputPath(job, new Path(ourArgs[1]));
+
+        //Suppression du fichier de sortie s'il existe déjà
+        FileSystem fs = FileSystem.newInstance(getConf());
+        if (fs.exists(outputFilePath)) {
+            fs.delete(outputFilePath, true);
+        }
+
+        return job.waitForCompletion(true) ? 0: 1;
+    }
+
+    public static void main(String[] args) throws Exception {
+        ExempleMapReduce exempleDriver = new ExempleMapReduce();
+        int res = ToolRunner.run(exempleDriver, args);
+        System.exit(res);
+    }
+}

Openclassrooms/Big-Data/text.txt

@@ -0,0 +1 @@
+1 2 1 2 this is a test

Openclassrooms/Big-Data/wordcount.py

@@ -0,0 +1,15 @@
+#! /usr/bin/env python
+
+from pyspark import SparkContext
+
+sc = SparkContext()
+
+lines = sc.textFile("/Users/marcpartensky/Programs/Python/Machine-Learning/Openclassrooms/Big-Data/text.txt")
+
+word_counts = lines.flatMap(lambda line: line.split(' ')) \
+        .map(lambda word: (word, 1)) \
+        .reduceByKey(lambda count1, count2: count1+count2) \
+        .collect()
+
+for (word, count) in word_counts:
+    print(word.encode("utf8"), count)

Sklearn/test.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import PoissonRegressor
+from sklearn.experimental import enable_hist_gradient_boosting  # noqa
+from sklearn.ensemble import HistGradientBoostingRegressor
+
+n_samples, n_features = 1000, 20
+rng = np.random.RandomState(0)
+X = rng.randn(n_samples, n_features)
+# positive integer target correlated with X[:, 5] with many zeros:
+y = rng.poisson(lam=np.exp(X[:, 5]) / 2)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng)
+glm = PoissonRegressor()
+gbdt = HistGradientBoostingRegressor(loss='poisson', learning_rate=.01)
+glm.fit(X_train, y_train)
+gbdt.fit(X_train, y_train)
+print(glm.score(X_test, y_test))
+print(gbdt.score(X_test, y_test))

Sklearn/test2.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+from sklearn import set_config
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+from sklearn.impute import SimpleImputer
+from sklearn.compose import make_column_transformer
+from sklearn.linear_model import LogisticRegression
+set_config(display='diagram')
+
+num_proc = make_pipeline(SimpleImputer(strategy='median'), StandardScaler())
+
+cat_proc = make_pipeline(
+    SimpleImputer(strategy='constant', fill_value='missing'),
+    OneHotEncoder(handle_unknown='ignore'))
+
+preprocessor = make_column_transformer((num_proc, ('feat1', 'feat3')),
+                                       (cat_proc, ('feat0', 'feat2')))
+
+clf = make_pipeline(preprocessor, LogisticRegression())
+clf

Sklearn/test3.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+import scipy
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.cluster import KMeans
+from sklearn.datasets import make_blobs
+from sklearn.metrics import completeness_score
+
+rng = np.random.RandomState(0)
+X, y = make_blobs(random_state=rng)
+X = scipy.sparse.csr_matrix(X)
+X_train, X_test, _, y_test = train_test_split(X, y, random_state=rng)
+kmeans = KMeans(algorithm='elkan').fit(X_train)
+print(completeness_score(kmeans.predict(X_test), y_test))