17 lines
376 B
Python
Executable File
17 lines
376 B
Python
Executable File
#! /usr/bin/env python
|
|
|
|
from pyspark import SparkContext
|
|
import sys
|
|
|
|
sc = SparkContext()
|
|
|
|
lines = sc.textFile(sys.argv[1])
|
|
|
|
word_counts = lines.flatMap(lambda line: line.split(' ')) \
|
|
.map(lambda word: (word, 1)) \
|
|
.reduceByKey(lambda count1, count2: count1+count2) \
|
|
.collect()
|
|
|
|
for (word, count) in word_counts:
|
|
print(word.encode("utf8"), count)
|