#!/usr/bin/env python3

"""word_count100.py"""
import sys
import os
import pyspark
from pyspark import SparkContext
import logging
from logging.config import fileConfig
import argparse


parser = argparse.ArgumentParser()
parser.add_argument("--master_ip", type=str,
                    help="IP address of the master node", required=True)
parser.add_argument("--path_logging_config", type=str,
                    help="Path to python logging config file in remote \
                    machine",
                    required=True)
args = parser.parse_args()

python_log_path = os.path.join(os.environ['SPARK_MASTER_LOG'],
                               'python_master.log')
fileConfig(args.path_logging_config,
           defaults={'logfile_path': python_log_path})
logger = logging.getLogger()

logger.debug("Launches the app.")

conf = pyspark.SparkConf().set("spark.driver.host", args.master_ip)

logger.debug("Creation of spark context")

sc = SparkContext(conf=conf)
lines = sc.textFile("/job/iliad100.txt", minPartitions=6)

word_counts = lines.flatMap(lambda line: line.split(' ')) \
                    .count()

logger.debug("Write the results in the output file.")

my_file = open("/job/output_iliad100.txt", "w")
my_file.write(str(word_counts))
my_file.close()

logger.debug("Inspect /job content:")
for x in os.listdir("/job"):
    logger.debug(x)


logger.debug("Closes the app.")

# for visualization on port 4040
# input("press ctrl+c to exit")
