Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import json
import os
import re
import shutil
import unicodedata
from functools import cache
import jsonschema
import mozpack.path as mozpath
from mozpack.files import FileFinder
@cache
def interventions_schema():
schema_path = mozpath.join(
os.path.dirname(__file__),
"intervention_schema.json",
)
with open(schema_path) as schema_fd:
return json.load(schema_fd)
def load_intervention_json(json_fd):
try:
config = json.load(json_fd)
except json.decoder.JSONDecodeError as e:
raise ValueError(f"{mozpath.basename(json_fd.path)} is invalid JSON: {e}")
try:
jsonschema.validate(instance=config, schema=interventions_schema())
except jsonschema.exceptions.ValidationError as e:
raise ValueError(
f"{mozpath.basename(json_fd.path)} is invalid intervention JSON: {e}"
)
return config
def clear_dir(path):
if not os.path.isdir(path) and not os.path.islink(path):
raise Exception(f"{path} is not a directory")
for filename in os.listdir(path):
file_path = mozpath.join(path, filename)
try:
if os.path.isfile(file_path) or os.path.islink(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(f"Failed to delete {file_path}. Reason: {e}")
def safe_filename(raw):
subbed = re.sub(
"_+",
"_",
raw
.replace(" ", "_")
.replace("\\", "_")
.replace("/", "_")
.replace(os.path.sep, "_")
.replace("(", "")
.replace(")", ""),
)
normalized = unicodedata.normalize("NFD", subbed)
return "".join([c for c in normalized if not unicodedata.combining(c)])
def maybe_lstrip(str, value):
if str.startswith(str):
return str[len(value) :].lstrip()
return str
def clean_script_template(script, filename):
# drop any license header, linter globals line, and "use strict".
script = script.rstrip()
while True:
script = script.lstrip()
if script.startswith("/*"):
script = script.partition("*/")[2]
elif script.startswith("//"):
script = script.partition("\n")[2]
elif script.startswith('"use strict";'):
script = script[13:]
else:
break
if not script:
raise ValueError(f"{filename} template does not seem to be a proper template")
if (
not script.startswith("{")
and not script.startswith("try")
and not script.startswith("if")
):
script = "\n ".join(script.splitlines())
script = f"{{\n {script}\n}}"
return script
SPECIAL_META_KEYS = ["all_frames", "match_origin_as_fallback", "user_styles"]
class special_js_script_checker:
def check(self, intervention, src_json_filename):
data = intervention.pop(self.section, None)
if data is None:
return None, None
metas = {name: False for name in SPECIAL_META_KEYS}
if type(data) is dict:
for key in SPECIAL_META_KEYS:
metas[key] = data.get(key, False)
return metas, self._get_params_from_data(data, src_json_filename)
class check_hide_alerts_section(special_js_script_checker):
def __init__(self):
self.section = "hide_alerts"
self.json_key = "alerts"
self.source = "hide_alerts.js"
def _get_params_from_data(self, data, src_json_filename):
if type(data) is list:
alertsToHide = data
elif type(data) is dict and "alerts" in data:
alertsToHide = data["alerts"]
else:
raise ValueError(
f"Unexpected data in {self.section} in {src_json_filename}: {str(data)}"
)
for alert in alertsToHide:
if alert.lower() != alert:
raise ValueError(
f"Please use lowercase values for {self.section} values (not `{alert}`) in {src_json_filename}"
)
return {"alertsToHide": alertsToHide}
class check_hide_messages_section(special_js_script_checker):
def __init__(self):
self.section = "hide_messages"
self.json_key = "messages"
self.source = "hide_messages.js"
def _get_params_from_data(self, data, src_json_filename):
if type(data) is list:
messagesToHide = data
elif type(data) is dict and "message" in data:
messagesToHide = [data]
elif type(data) is dict and "messages" in data:
messagesToHide = data["messages"]
else:
raise ValueError(
f"Unexpected data in {self.section} in {src_json_filename}: {str(data)}"
)
return {"messagesToHide": messagesToHide}
class check_modify_meta_viewport_section(special_js_script_checker):
def __init__(self):
self.section = "modify_meta_viewport"
self.json_key = "modify"
self.source = "modify_meta_viewport.js"
def _get_params_from_data(self, data, src_json_filename):
if type(data) is dict and "modify" in data:
metaViewportChanges = data["modify"]
elif type(data) is dict:
metaViewportChanges = data
else:
raise ValueError(
f"Unexpected data in {self.section} in {src_json_filename}: {str(data)}"
)
return {"metaViewportChanges": metaViewportChanges}
def bake_params_into_script_template(template, params={}):
for name, value in params.items():
template = template.replace(
f'"param:{name}"', json.dumps(value, sort_keys=True)
)
return template
@cache
def get_script_template(filename, *dirs_to_try):
for dir in dirs_to_try:
path_to_try = mozpath.join(dir, filename)
if os.path.isfile(path_to_try) and os.access(path_to_try, os.R_OK):
with open(path_to_try) as template_fd:
return clean_script_template(template_fd.read(), filename)
raise ValueError(
f"Could not access expected template {filename} in {' or '.join(dirs_to_try)}"
)
def build_logger_script(config, templates_dir):
domain_to_bug_numbers = {}
for bug_number, data in config.get("bugs", {}).items():
for match in data.get("matches", []):
domain = match.partition("://")[2].split("/")[0].replace("*.", "")
domain_to_bug_numbers.setdefault(domain, set())
domain_to_bug_numbers[domain].add(bug_number)
bugInfo = [[k, sorted(list(v))] for k, v in domain_to_bug_numbers.items()]
return bake_params_into_script_template(
get_script_template("log_console_message.js", templates_dir),
{"bugInfo": bugInfo},
)
def determine_generated_content_scripts_for_intervention(
config, bug_number, src_json_filename, interventions_dir
):
injections_dir = mozpath.normpath(
mozpath.join(
interventions_dir,
"..",
"..",
"injections",
)
)
return determine_generated_css_content_scripts_for_intervention(
config, bug_number, src_json_filename
) | determine_generated_js_content_scripts_for_intervention(
config, bug_number, src_json_filename, injections_dir
)
def determine_generated_js_content_scripts_for_intervention(
config, bug_number, src_json_filename, injections_dir
):
# The JSON files for interventions may used generic JS scripts, including
# special ones with special info in the JSON, like this:
#
# "interventions": [
# {
# "hide_messages": [{ "container": ".header.caution", "message": "unsupported browser"}],
# "hide_alerts": { "all_frames": true, "alerts: ["Chrome"] },
# "modify_meta_viewport": {
# "interactive-widget": "resizes-content",
# },
# "content_scripts": {
# "js": ["use_chrome_useragent.js"]
# }
# },
#
# We want to combine these into one final content-scripts:
#
# {
# "content_scripts": {
# "js": ["injections/generated/bug12345_whatever.com.js"]
# }
# },
files_to_generate = {}
generated_filenames_cache = {}
special_checkers = [
check_hide_alerts_section(),
check_hide_messages_section(),
check_modify_meta_viewport_section(),
]
label = safe_filename(config["label"])
for intervention in config["interventions"]:
final_metas = None
content_scripts = intervention.get("content_scripts", {})
if content_scripts:
final_metas = {}
for key in SPECIAL_META_KEYS:
final_metas[key] = content_scripts.get(key, False)
js = content_scripts.get("js", [])
generate_from_sources = []
if js and any(not script_filename.startswith("bug") for script_filename in js):
generate_from_sources = [
{
"params": {},
"source": script_filename,
}
for script_filename in js
]
for checker in special_checkers:
metas, params = checker.check(intervention, src_json_filename)
if not metas and not params:
continue
if final_metas is None:
final_metas = metas
elif final_metas != metas:
raise ValueError(
f"cannot mix true/false values of {SPECIAL_META_KEYS} in the same intervention in {src_json_filename}"
)
generate_from_sources.append({
"params": params,
"source": checker.source,
})
if generate_from_sources:
cache_key = json.dumps(generate_from_sources, sort_keys=True)
generated_filename = generated_filenames_cache.get(cache_key, None)
if not generated_filename:
suffix = ""
next_generated_script_num = len(generated_filenames_cache)
if next_generated_script_num:
suffix = f"-{next_generated_script_num}"
generated_filename = safe_filename(
f"bug{bug_number}-{label}{suffix}.js"
)
generated_filenames_cache[cache_key] = generated_filename
content_scripts = intervention.setdefault("content_scripts", {})
js = content_scripts.setdefault("js", [])
content_scripts["js"] = [f"injections/generated/{generated_filename}"]
for name, value in final_metas.items():
if value:
content_scripts[name] = True
files_to_generate[generated_filename] = generate_from_sources
return files_to_generate
def determine_generated_css_content_scripts_for_intervention(
config, bug_number, src_json_filename
):
# The JSON files for interventions may contain css sections like this:
#
# "label": "whatever.com",
# "css": {
# "fix_broken_slider": "css text 1",
# "remove_extra_scrollbars": "css text 2"
# },
# "interventions": [
# {
# "css": ["fix_broken_slider"]
# },
# {
# "css": {
# "all_frames": true,
# "match_origin_as_fallback": true,
# "user_styles": true, // cssOrigin = "user" (otherwise is default of "author")
# "which": ["fix_broken_slider", "remove_extra_scrollbars"]
# }
# }
# ]
#
# These must be replaced with corresponding content_scripts sections while building the
# final run.js (and the files it references must also be generated):
#
# "label": "whatever.com",
# "interventions": [
# {
# "content_scripts": {
# "css": ["injections/generated/bug12345_whatever.com_fix_broken_slider.css"]
# }
# },
# {
# "content_scripts": {
# "all_frames": true,
# "match_origin_as_fallback": true,
# "css": ["injections/generated/bug12345_whatever.com_fix_broken_slider.css",
# "injections/generated/bug12345_whatever.com_remove_extra_scrollbars.css"]
# }
# }
# ]
files_to_generate = {}
if "css" in config and (type(config["css"]) is not dict or not config["css"]):
raise ValueError(
f"css section should be a non-empty object or be removed from {src_json_filename}"
)
css_files = config.pop("css", None)
if not css_files:
for intervention in config["interventions"]:
if intervention.get("css"):
raise ValueError(
f"css wanted, but none specified for {src_json_filename}"
)
return files_to_generate
actually_used_files = set()
for cssText in css_files.values():
if type(cssText) is not str or not cssText:
raise ValueError(
f"css text should be a non-empty string in {src_json_filename}"
)
label = safe_filename(config["label"])
for intervention in config["interventions"]:
css = intervention.pop("css", None)
if css is None:
continue
if type(css) is list:
css = {"which": css}
elif type(css) is not dict:
raise ValueError(
f"css sections should be a non-empty object or list or be removed from interventions in {src_json_filename}"
)
which_css_files_to_add = css.pop("which", None)
if not which_css_files_to_add or type(which_css_files_to_add) is not list:
raise ValueError(
f"intervention with missing `which` key or invalid array of desired css files in {src_json_filename}"
)
for file in which_css_files_to_add:
if type(file) is not str or not file:
raise ValueError(
f"Empty or non-string filename not listed in intervention css section of {src_json_filename}"
)
if not css_files.get(file):
raise ValueError(
f"{file} is not listed in css section of {src_json_filename}"
)
actually_used_files.add(file)
metas = {}
for key in SPECIAL_META_KEYS:
metas[key] = css.pop(key, False)
if not isinstance(metas[key], bool):
raise ValueError(
f"{key} must be `true` or `false` in {src_json_filename}"
)
unknown_keys = "','".join(css.keys())
if unknown_keys:
raise ValueError(
f"unknown key(s) '{unknown_keys}' in css section of {src_json_filename}"
)
content_scripts = intervention.get("content_scripts", None)
if content_scripts:
for key in SPECIAL_META_KEYS:
if content_scripts.get(key, False) != metas[key]:
raise ValueError(
f"cannot mix value of {key} in css and content_scripts sections in {src_json_filename}"
)
content_scripts = intervention.setdefault("content_scripts", {})
css = content_scripts.setdefault("css", [])
for filename in which_css_files_to_add:
final_filename = safe_filename(f"bug{bug_number}-{label}-{filename}.css")
css.append(f"injections/generated/{final_filename}")
files_to_generate[final_filename] = [{"contents": css_files[filename]}]
for key in SPECIAL_META_KEYS:
if metas[key]:
content_scripts[key] = True
extras = set(css_files.keys()).difference(actually_used_files)
if extras:
raise ValueError(
f"Extra css fragments specified which aren't used in {src_json_filename}: "
+ ", ".join(list(extras))
)
return files_to_generate
def generate_run_js(
output_fd,
template_path,
interventions_dir,
*_preprocessed_intervention_files_mozbuild,
):
preprocessed_intervention_files_mozbuild = [
f
for f in _preprocessed_intervention_files_mozbuild
if not f.endswith("/codegen.py")
]
with open(template_path) as template_fd:
input_files = list(FileFinder(interventions_dir).find("*.json"))
filenames_json_files_expect_to_generate = set()
final_interventions = {}
actually_referenced_non_generated_files = set()
for json_filename, json_fd in input_files:
bug_number = mozpath.splitext(mozpath.basename(json_filename))[0].split(
"-"
)[0]
config = load_intervention_json(json_fd)
final_interventions[bug_number] = config
# Do some sanity checks first
listed_bugs = config.get("bugs", {}).keys()
if len(listed_bugs) < 2 and bug_number.split("_")[0] not in listed_bugs:
raise ValueError(
f"Bug number in the filename ({bug_number}) does not match bugs section of {json_filename}"
)
for intervention in config["interventions"]:
content_scripts = intervention.get("content_scripts", {})
listed_files = set()
for type in ["css", "js"]:
for non_generated_filename in content_scripts.get(type, []):
actually_referenced_non_generated_files.add(
non_generated_filename
)
if f"injections/{type}/" in non_generated_filename:
raise ValueError(
f"Please remove the unneeded 'injections/{type}/' from '{non_generated_filename}' intervention in {json_filename}"
)
if not non_generated_filename.endswith(f".{type}"):
raise ValueError(
f"{non_generated_filename} does not end in .{type} in {json_filename}"
)
if non_generated_filename in listed_files:
raise ValueError(
f"{non_generated_filename} is listed twice in same intervention in {json_filename}"
)
listed_files.add(non_generated_filename)
actual_path = mozpath.normpath(
mozpath.join(
interventions_dir,
"..",
"..",
"injections",
type,
non_generated_filename,
)
)
if not os.path.isfile(actual_path) or not os.access(
actual_path, os.R_OK
):
raise ValueError(
f"{non_generated_filename} is not an accessible file in {json_filename} (expected at {actual_path})"
)
if os.path.splitext(actual_path)[1] != "." + type:
raise ValueError(
f"File extension for {actual_path} should be .{type} in {json_filename}"
)
# Now remove each css section in this JSON and ensure that a corresponding
# content_scripts section exists, with the files named as they will be after
# they are generated by generate_css_intervention. Also double-check that we
# will not be stomping over any already-existing non-generated files.
generated_files = determine_generated_content_scripts_for_intervention(
config, bug_number, json_filename, interventions_dir
)
for filename in generated_files:
filenames_json_files_expect_to_generate.add(filename)
# Halt if preprocessed_intervention_files.mozbuild needs to be updated
filenames_listed_in_mozbuild = set(preprocessed_intervention_files_mozbuild)
extra_generated_files = sorted(
filenames_json_files_expect_to_generate.difference(
filenames_listed_in_mozbuild
)
)
missing_generated_files = sorted(
filenames_listed_in_mozbuild.difference(
filenames_json_files_expect_to_generate
)
)
if extra_generated_files or missing_generated_files:
msg = ""
if missing_generated_files:
msg += "\nPlease remove: " + ", ".join(missing_generated_files)
if extra_generated_files:
msg += "\nPlease add: " + ", ".join(extra_generated_files)
raise ValueError(
"preprocessed_intervention_files.mozbuild is out of date:" + msg
)
# Check if any non-generated css/js files aren't being used anymore.
non_generated_files_path = mozpath.normpath(
mozpath.join(interventions_dir, "..", "..", "injections")
)
non_generated_files = set(
name
for name, _ in FileFinder(
mozpath.join(non_generated_files_path, "css")
).find("*")
)
non_generated_files.update(
set(
name
for name, _ in FileFinder(
mozpath.join(non_generated_files_path, "js")
).find("*")
)
)
extra_non_generated_files = sorted(
non_generated_files.difference(actually_referenced_non_generated_files)
)
# Generic css/js files are bundled in case we need them for remote updates,
# so don't ask for them to be removed even if they seem unreferenced.
extra_non_generated_files = [
filename
for filename in extra_non_generated_files
if filename.startswith("bug")
]
if extra_non_generated_files:
raise ValueError(
"Please remove these files which are not referenced in any intervention JSON file: "
+ ", ".join(extra_non_generated_files)
)
# Emit the final run.json
interventions_json = json.dumps(
dict(sorted(final_interventions.items())), indent=2, sort_keys=True
)
raw = template_fd.read()
subbed = raw.replace(
"// Note that this variable is expanded during build-time. See bz2019069 for details.\n",
"",
)
subbed = raw.replace(
"AVAILABLE_INTERVENTIONS = {}",
f"AVAILABLE_INTERVENTIONS = {interventions_json}",
)
output_fd.write(subbed)
def generate_file(outfile, interventions_dir, *ignored):
desired_filename = mozpath.basename(outfile.name)
is_css = outfile.name.endswith(".css")
is_js = outfile.name.endswith(".js")
if not is_css and not is_js:
raise ValueError(f"Do not know how to generate {outfile.name}")
bug_number = (
mozpath
.splitext(mozpath.basename(desired_filename))[0]
.split("-")[0]
.lstrip("bug")
)
expected_json_filename = "-".join(
mozpath.splitext(desired_filename.lstrip("bug"))[0].split("-")[0:2]
)
json_files = list(
FileFinder(interventions_dir).find(f"{expected_json_filename}*.json")
)
if not json_files:
raise ValueError(
f"no json intervention file starting with {expected_json_filename}"
)
if len(json_files) > 1:
json_files = " or ".join(f[0] for f in json_files)
raise ValueError(
f"multiple json intervention files starting with {expected_json_filename}.. not sure which to use to generate {desired_filename} from {json_files}"
)
json_filename, json_fd = json_files[0]
config = load_intervention_json(json_fd)
generated_files = determine_generated_content_scripts_for_intervention(
config, bug_number, json_filename, interventions_dir
)
needed_parts = generated_files.get(desired_filename, None)
if needed_parts is None or not needed_parts:
raise ValueError(
f"No needed parts found to generate {desired_filename} for {json_filename}"
)
generated_parts = []
templates_dir = mozpath.normpath(
mozpath.join(
interventions_dir,
"..",
"..",
"templates",
)
)
js_injections_dir = mozpath.normpath(
mozpath.join(templates_dir, "..", "injections", "js")
)
for part_info in needed_parts:
if part_info.get("contents", None):
# css files go here
generated_parts.append(part_info["contents"])
continue
params = part_info["params"]
source_filename = part_info["source"]
template = get_script_template(
source_filename, templates_dir, js_injections_dir
)
generated_parts.append(bake_params_into_script_template(template, params))
if not generated_parts:
raise ValueError(
f"Failed to generate anything for {outfile.name} in {json_filename}"
)
all_parts = "\n\n".join(generated_parts)
if is_js:
if "__webcompat_spoof_platform" in all_parts:
all_parts += "\n\ndelete window.__webcompat_spoof_platform;"
if "window.__webcompat" in all_parts.replace("__webcompat_spoof_platform", ""):
all_parts += "\n\n" + build_logger_script(config, templates_dir)
outfile.write(
"/* THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. */\n\n"
)
outfile.write(all_parts)
def main(*args): # mach requires this
pass