"""

return {

"numberofrules": 0,

"datapath": path_join_robust(BASEDIR_PATH, "data"),

"freshen": True,

"replace": False,

"backup": False,

"skipstatichosts": False,

"keepdomaincomments": False,

"extensionspath": path_join_robust(BASEDIR_PATH, "extensions"),

"extensions": [],

"compress": False,

"minimise": False,

"outputsubfolder": "",

"hostfilename": "hosts",

"targetip": "0.0.0.0",

"sourcedatafilename": "update.json",

"sourcesdata": [],

"readmefilename": "readme.md",

"readmetemplate": path_join_robust(BASEDIR_PATH, "readme_template.md"),

"readmedata": {},

"readmedatafilename": path_join_robust(BASEDIR_PATH, "readmeData.json"),

"exclusionpattern": "([a-zA-Z\d-]+\.){0,}",

"exclusionregexs": [],

"exclusions": [],

"commonexclusions": ["hulu.com"],

"blacklistfile": path_join_robust(BASEDIR_PATH, "blacklist"),

parser = argparse.ArgumentParser(description="Creates a unified hosts "

"file from hosts stored in "

"data subfolders.")

parser.add_argument("--auto", "-a", dest="auto", default=False,

action="store_true", help="Run without prompting.")

parser.add_argument("--backup", "-b", dest="backup", default=False,

action="store_true", help="Backup the hosts "

"files before they "

"are overridden.")

parser.add_argument("--extensions", "-e", dest="extensions", default=[],

nargs="*", help="Host extensions to include "

"in the final hosts file.")

parser.add_argument("--ip", "-i", dest="targetip", default="0.0.0.0",

help="Target IP address. Default is 0.0.0.0.")

parser.add_argument("--keepdomaincomments", "-k",

dest="keepdomaincomments", default=False,

help="Keep domain line comments.")

parser.add_argument("--noupdate", "-n", dest="noupdate", default=False,

action="store_true", help="Don't update from "

"host data sources.")

parser.add_argument("--skipstatichosts", "-s", dest="skipstatichosts",

default=False, action="store_true",

help="Skip static localhost entries "

"in the final hosts file.")

parser.add_argument("--output", "-o", dest="outputsubfolder", default="",

help="Output subfolder for generated hosts file.")

parser.add_argument("--replace", "-r", dest="replace", default=False,

action="store_true", help="Replace your active "

"hosts file with this "

"new hosts file.")

parser.add_argument("--flush-dns-cache", "-f", dest="flushdnscache",

default=False, action="store_true",

help="Attempt to flush DNS cache "

"after replacing the hosts file.")

parser.add_argument("--compress", "-c", dest="compress",

default=False, action="store_true",

help="Compress the hosts file "

"ignoring non-necessary lines "

"(empty lines and comments) and "

"putting multiple domains in "

"each line. Improve the "

"performances under Windows.")

parser.add_argument("--minimise", "-m", dest="minimise",

default=False, action="store_true",

help="Minimise the hosts file "

"ignoring non-necessary lines "

"(empty lines and comments).")

global settings

options = vars(parser.parse_args())

options["outputpath"] = path_join_robust(BASEDIR_PATH, options["outputsubfolder"])

options["freshen"] = not options["noupdate"]

settings = get_defaults()

settings.update(options)

data_path = settings["datapath"]

extensions_path = settings["extensionspath"]

settings["sources"] = list_dir_no_hidden(data_path)

settings["extensionsources"] = list_dir_no_hidden(extensions_path)

# All our extensions folders...

settings["extensions"] = [os.path.basename(item) for item in list_dir_no_hidden(extensions_path)]

# ... intersected with the extensions passed-in as arguments, then sorted.

settings["extensions"] = sorted(list(

set(options["extensions"]).intersection(settings["extensions"])))

auto = settings["auto"]

exclusion_regexes = settings["exclusionregexs"]

source_data_filename = settings["sourcedatafilename"]

update_sources = prompt_for_update(freshen=settings["freshen"],

update_auto=auto)

if update_sources:

update_all_sources(source_data_filename, settings["hostfilename"])

gather_exclusions = prompt_for_exclusions(skip_prompt=auto)

if gather_exclusions:

common_exclusions = settings["commonexclusions"]

exclusion_pattern = settings["exclusionpattern"]

exclusion_regexes = display_exclusion_options(

common_exclusions=common_exclusions,

exclusion_pattern=exclusion_pattern,

exclusion_regexes=exclusion_regexes)

extensions = settings["extensions"]

sources_data = update_sources_data(settings["sourcesdata"],

datapath=data_path,

extensions=extensions,

extensionspath=extensions_path,

sourcedatafilename=source_data_filename)

merge_file = create_initial_file()

remove_old_hosts_file(settings["backup"])

if settings["compress"]:

# Another mode is required to read and write the file in Python 3

final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b" if PY3 else "w+")

compressed_file = tempfile.NamedTemporaryFile()

remove_dups_and_excl(merge_file, exclusion_regexes, compressed_file)

compress_file(compressed_file, settings["targetip"], final_file)

elif settings["minimise"]:

final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b" if PY3 else "w+")

minimised_file = tempfile.NamedTemporaryFile()

remove_dups_and_excl(merge_file, exclusion_regexes, minimised_file)

minimise_file(minimised_file, settings["targetip"], final_file)

else:

final_file = remove_dups_and_excl(merge_file, exclusion_regexes)

number_of_rules = settings["numberofrules"]

output_subfolder = settings["outputsubfolder"]

skip_static_hosts = settings["skipstatichosts"]

write_opening_header(final_file, extensions=extensions,

numberofrules=number_of_rules,

outputsubfolder=output_subfolder,

skipstatichosts=skip_static_hosts)

final_file.close()

update_readme_data(settings["readmedatafilename"],

extensions=extensions,

numberofrules=number_of_rules,

outputsubfolder=output_subfolder,

sourcesdata=sources_data)

print_success("Success! The hosts file has been saved in folder " +

output_subfolder + "\nIt contains " +

"{:,}".format(number_of_rules) +

" unique entries.")

move_file = prompt_for_move(final_file, auto=auto,

replace=settings["replace"],

skipstatichosts=skip_static_hosts)

# We only flush the DNS cache if we have

# moved a new hosts file into place.

if move_file:

prompt_for_flush_dns_cache(flush_cache=settings["flushdnscache"],

"""

# Create a hosts file if it doesn't exist.

hosts_file = path_join_robust(BASEDIR_PATH, "hosts")

if not os.path.isfile(hosts_file):

try:

open(hosts_file, "w+").close()

except (IOError, OSError):

# Starting in Python 3.3, IOError is aliased

# OSError. However, we have to catch both for

# Python 2.x failures.

print_failure("ERROR: No 'hosts' file in the folder. Try creating one manually.")

if not freshen:

return

prompt = "Do you want to update all data sources?"

if update_auto or query_yes_no(prompt):

return True

elif not update_auto:

print("OK, we'll stick with what we've got locally.")

"""

prompt = ("Do you want to exclude any domains?\n"

"For example, hulu.com video streaming must be able to access "

"its tracking and ad servers in order to play video.")

if not skip_prompt:

if query_yes_no(prompt):

return True

else:

print("OK, we'll only exclude domains in the whitelist.")

"""

if flush_cache:

flush_dns_cache()

elif prompt_flush:

if query_yes_no("Attempt to flush the DNS cache?"):

"""

skip_static_hosts = move_params["skipstatichosts"]

if move_params["replace"] and not skip_static_hosts:

move_file = True

elif move_params["auto"] or skip_static_hosts:

move_file = False

else:

prompt = ("Do you want to replace your existing hosts file with the newly generated file?")

move_file = query_yes_no(prompt)

if move_file:

move_hosts_file_into_place(final_file)

"""

for exclusion_option in common_exclusions:

prompt = "Do you want to exclude the domain " + exclusion_option + " ?"

if query_yes_no(prompt):

exclusion_regexes = exclude_domain(exclusion_option,

exclusion_pattern,

exclusion_regexes)

else:

continue

if query_yes_no("Do you want to exclude any other domains?"):

exclusion_regexes = gather_custom_exclusions(exclusion_pattern,

exclusion_regexes)

"""

# We continue running this while-loop until the user

# says that they have no more domains to exclude.

while True:

domain_prompt = ("Enter the domain you want to exclude (e.g. facebook.com): ")

user_domain = raw_input(domain_prompt)

if is_valid_domain_format(user_domain):

exclusion_regexes = exclude_domain(user_domain, exclusion_pattern, exclusion_regexes)

continue_prompt = "Do you have more domains you want to enter?"

if not query_yes_no(continue_prompt):

break

"""

exclusion_regex = re.compile(exclusion_pattern + domain)

exclusion_regexes.append(exclusion_regex)

"""

stripped_domain = stripped_rule.split()[1]

for exclusionRegex in exclusion_regexes:

if exclusionRegex.search(stripped_domain):

return True

"""

source_data_filename = sources_params["sourcedatafilename"]

for source in recursive_glob(sources_params["datapath"], source_data_filename):

update_file = open(source, "r")

update_data = json.load(update_file)

sources_data.append(update_data)

update_file.close()

for source in sources_params["extensions"]:

source_dir = path_join_robust(

sources_params["extensionspath"], source)

for update_file_path in recursive_glob(source_dir, source_data_filename):

update_file = open(update_file_path, "r")

update_data = json.load(update_file)

sources_data.append(update_data)

update_file.close()

"""

temp_list = json.loads(json_array_string)

hostlines = "127.0.0.1 " + "\n127.0.0.1 ".join(temp_list)

"""

# The transforms we support

transform_methods = {

'jsonarray': jsonarray

}

all_sources = recursive_glob("*", source_data_filename)

for source in all_sources:

update_file = open(source, "r")

update_data = json.load(update_file)

update_file.close()

update_url = update_data["url"]

update_transforms = []

if update_data.get("transforms"):

update_transforms = update_data["transforms"]

print("Updating source " + os.path.dirname(source) + " from " + update_url)

try:

updated_file = get_file_by_url(update_url)

# spin the transforms as required

for transform in update_transforms:

updated_file = transform_methods[transform](updated_file)

# get rid of carriage-return symbols

updated_file = updated_file.replace("\r", "")

hosts_file = open(path_join_robust(BASEDIR_PATH,

os.path.dirname(source),

host_filename), "wb")

write_data(hosts_file, updated_file)

hosts_file.close()

except Exception:

"""

merge_file = tempfile.NamedTemporaryFile()

# spin the sources for the base file

for source in recursive_glob(settings["datapath"],

settings["hostfilename"]):

with open(source, "r") as curFile:

write_data(merge_file, curFile.read())

# spin the sources for extensions to the base file

for source in settings["extensions"]:

for filename in recursive_glob(path_join_robust(

settings["extensionspath"], source), settings["hostfilename"]):

with open(filename, "r") as curFile:

write_data(merge_file, curFile.read())

if os.path.isfile(settings["blacklistfile"]):

with open(settings["blacklistfile"], "r") as curFile:

write_data(merge_file, curFile.read())

"""

input_file.seek(0) # reset file pointer

write_data(output_file, '\n')

target_ip_len = len(target_ip)

lines = [target_ip]

lines_index = 0

for line in input_file.readlines():

line = line.decode("UTF-8")

if line.startswith(target_ip):

if lines[lines_index].count(' ') < 9:

lines[lines_index] += ' ' \

+ line[target_ip_len:line.find('#')].strip()

else:

lines[lines_index] += '\n'

lines.append(line[:line.find('#')].strip())

lines_index += 1

for line in lines:

write_data(output_file, line)

"""

input_file.seek(0) # reset file pointer

write_data(output_file, '\n')

lines = []

for line in input_file.readlines():

line = line.decode("UTF-8")

if line.startswith(target_ip):

lines.append(line[:line.find('#')].strip() + '\n')

for line in lines:

write_data(output_file, line)

"""

number_of_rules = settings["numberofrules"]

if os.path.isfile(settings["whitelistfile"]):

with open(settings["whitelistfile"], "r") as ins:

for line in ins:

line = line.strip(" \t\n\r")

if line and not line.startswith("#"):

settings["exclusions"].append(line)

if not os.path.exists(settings["outputpath"]):

os.makedirs(settings["outputpath"])

if output_file is None:

# Another mode is required to read and write the file in Python 3

final_file = open(path_join_robust(settings["outputpath"], "hosts"), "w+b" if PY3 else "w+")

else:

final_file = output_file

merge_file.seek(0) # reset file pointer

hostnames = {"localhost", "localhost.localdomain", "local", "broadcasthost"}

exclusions = settings["exclusions"]

for line in merge_file.readlines():

write_line = True

# Explicit encoding

line = line.decode("UTF-8")

# replace tabs with space

line = line.replace("\t+", " ")

# see gh-271: trim trailing whitespace, periods

line = line.rstrip(' .') + "\n"

# Testing the first character doesn't require startswith

if line[0] == "#" or re.match(r'^\s*$', line[0]):

write_data(final_file, line)

continue

if "::1" in line:

continue

stripped_rule = strip_rule(line) # strip comments

if not stripped_rule or matches_exclusions(stripped_rule,

exclusion_regexes):

continue

# Normalize rule

hostname, normalized_rule = normalize_rule(

stripped_rule, target_ip=settings["targetip"],

keep_domain_comments=settings["keepdomaincomments"])

for exclude in exclusions:

if re.search('[\s\.]' + re.escape(exclude) + '\s', line):

write_line = False

break

if normalized_rule and (hostname not in hostnames) and write_line:

write_data(final_file, normalized_rule)

hostnames.add(hostname)

number_of_rules += 1

settings["numberofrules"] = number_of_rules

merge_file.close()

if output_file is None:

"""

"""

regex = r'^\s*(\d{1,3}\.){3}\d{1,3}\s+([\w\.-]+[a-zA-Z])(.*)'

result = re.search(regex, rule)

if result:

hostname, suffix = result.group(2, 3)

# Explicitly lowercase and trim the hostname.

hostname = hostname.lower().strip()

rule = "%s %s" % (target_ip, hostname)

if suffix and keep_domain_comments:

rule += " #%s" % suffix

return hostname, rule + "\n"

"""

regex = r'^\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*(.*)'

result = re.search(regex, rule)

if result:

ip_host, suffix = result.group(2, 3)

# Explicitly trim the ip host.

ip_host = ip_host.strip()

rule = "%s %s" % (target_ip, ip_host)

if suffix and keep_domain_comments:

rule += " #%s" % suffix

return ip_host, rule + "\n"

"""

print("==>%s<==" % rule)

"""

split_line = line.split()

if len(split_line) < 2:

# just return blank

return ""

else:

"""

final_file.seek(0) # Reset file pointer.

file_contents = final_file.read() # Save content.

final_file.seek(0) # Write at the top.

write_data(final_file, "# This hosts file is a merged collection "

"of hosts from reputable sources,\n")

write_data(final_file, "# with a dash of crowd sourcing via Github\n#\n")

write_data(final_file, "# Date: " + time.strftime("%B %d %Y", time.gmtime()) + "\n")

if header_params["extensions"]:

write_data(final_file, "# Extensions added to this file: " + ", ".join(

header_params["extensions"]) + "\n")

write_data(final_file, ("# Number of unique domains: {:,}\n#\n".format(header_params["numberofrules"])))

write_data(final_file, "# Fetch the latest version of this file: "

"https://raw.githubusercontent.com/StevenBlack/hosts/master/" +

path_join_robust(header_params["outputsubfolder"], "") + "hosts\n")

write_data(final_file, "# Project home page: https://github.com/StevenBlack/hosts\n")

write_data(final_file, "# Project releases: https://github.com/StevenBlack/hosts/releases\n#\n")

write_data(final_file, "# ===============================================================\n")

write_data(final_file, "\n")

if not header_params["skipstatichosts"]:

write_data(final_file, "127.0.0.1 localhost\n")

write_data(final_file, "127.0.0.1 localhost.localdomain\n")

write_data(final_file, "127.0.0.1 local\n")

write_data(final_file, "255.255.255.255 broadcasthost\n")

write_data(final_file, "::1 localhost\n")

write_data(final_file, "::1 ip6-localhost\n")

write_data(final_file, "::1 ip6-loopback\n")

write_data(final_file, "fe80::1%lo0 localhost\n")

write_data(final_file, "ff00::0 ip6-localnet\n")

write_data(final_file, "ff00::0 ip6-mcastprefix\n")

write_data(final_file, "ff02::1 ip6-allnodes\n")

write_data(final_file, "ff02::2 ip6-allrouters\n")

write_data(final_file, "ff02::3 ip6-allhosts\n")

write_data(final_file, "0.0.0.0 0.0.0.0\n")

if platform.system() == "Linux":

write_data(final_file, "127.0.1.1 " + socket.gethostname() + "\n")

write_data(final_file, "127.0.0.53 " + socket.gethostname() + "\n")

write_data(final_file, "\n")

preamble = path_join_robust(BASEDIR_PATH, "myhosts")

if os.path.isfile(preamble):

with open(preamble, "r") as f:

write_data(final_file, f.read())

"""

extensions_key = "base"

extensions = readme_updates["extensions"]

if extensions:

extensions_key = "-".join(extensions)

output_folder = readme_updates["outputsubfolder"]

generation_data = {"location": path_join_robust(output_folder, ""),

"entries": readme_updates["numberofrules"],

"sourcesdata": readme_updates["sourcesdata"]}

with open(readme_file, "r") as f:

readme_data = json.load(f)

readme_data[extensions_key] = generation_data