cm_crowdin: optimize the script
PS1: Fix duplication of code
PS2: Fix commit info
Simplify code structure of commit creation
PS3: Fix wrong comparisons
PS4: Simplify commit creation further
Simplify get_caf_additions()
Add the default branch as a shared variable
Simplify the removal of empty translations
PS5: Fix whitespace error
PS6: Fix commit message
PS7: First steps to integrate JS website translations
PS8: First full support for JS translations
More comments
PS9: Determine default branch by looking it up in android/default.xml
PS10: Rebase
Rename script to 'crowdin_sync.py'
PS11: Add initial support for command line arguments
PS12: Rebase after latest changes
PS13: Fix arguments (parsing)
Fix determination of default branch
PS14: Additional fixes
PS15: Rebase & cleanup
PS16: Ready to merge
Change-Id: I1a1a108f7f67cb51cb27cc16f9f333e1e09a8520
diff --git a/crowdin/crowdin_aosp.yaml b/crowdin/crowdin_aosp.yaml
index 6601b1e..061cdf8 100644
--- a/crowdin/crowdin_aosp.yaml
+++ b/crowdin/crowdin_aosp.yaml
@@ -1,4 +1,4 @@
-# crowdin-aosp.yaml
+# crowdin_aosp.yaml
#
# Crowdin configuration file for CyanogenMod's
# additional languages not supported by AOSP
diff --git a/crowdin/crowdin_cm.yaml b/crowdin/crowdin_cm.yaml
index bcdbe77..f2da699 100644
--- a/crowdin/crowdin_cm.yaml
+++ b/crowdin/crowdin_cm.yaml
@@ -1,4 +1,4 @@
-# crowdin.yaml
+# crowdin_cm.yaml
#
# Crowdin configuration file for CyanogenMod
#
diff --git a/crowdin/js.xml b/crowdin/js.xml
new file mode 100644
index 0000000..abaf149
--- /dev/null
+++ b/crowdin/js.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Copyright (C) 2014 The CyanogenMod Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<js>
+ <project path="website/CMAccountWebsite/app/js/translations" source="en.js" />
+</js>
diff --git a/crowdin_sync.py b/crowdin_sync.py
index 319855d..f1dfe4d 100755
--- a/crowdin_sync.py
+++ b/crowdin_sync.py
@@ -19,6 +19,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+############################################# IMPORTS ##############################################
+
+import argparse
import codecs
import git
import os
@@ -30,6 +33,8 @@
from urllib import urlretrieve
from xml.dom import minidom
+############################################ FUNCTIONS #############################################
+
def get_caf_additions(strings_base, strings_cm):
# Load AOSP file and resources
xml_base = minidom.parse(strings_base)
@@ -42,26 +47,11 @@
list_cm_string_array = xml_cm.getElementsByTagName('string-array')
list_cm_plurals = xml_cm.getElementsByTagName('plurals')
- # All names from CM
- names_cm_string = []
- names_cm_string_array = []
- names_cm_plurals = []
- # All names from AOSP
+ # Load all names from AOSP
names_base_string = []
names_base_string_array = []
names_base_plurals = []
- # Get all names from CM
- for s in list_cm_string :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_string.append(s.attributes['name'].value)
- for s in list_cm_string_array :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_string_array.append(s.attributes['name'].value)
- for s in list_cm_plurals :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_plurals.append(s.attributes['name'].value)
- # Get all names from AOSP
for s in list_base_string :
if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
names_base_string.append(s.attributes['name'].value)
@@ -75,35 +65,26 @@
# Store all differences in this list
caf_additions = []
- # Store all found strings/arrays/plurals.
- # Prevent duplicates with product attribute
- found_string = []
- found_string_array = []
- found_plurals = []
+ # Loop through all CM resources. If an ID cannot be found in AOSP base file,
+ # the ID is from CAF and will be added to 'caf_additions'
+ for s in list_cm_string :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_string:
+ caf_additions.append(' ' + s.toxml())
+ for s in list_cm_string_array :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_string_array:
+ caf_additions.append(' ' + s.toxml())
+ for s in list_cm_plurals :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_plurals:
+ caf_additions.append(' ' + s.toxml())
- # Add all CAF additions to the list 'caf_additions'
- for z in names_cm_string:
- if z not in names_base_string and z not in found_string:
- for string_item in list_cm_string:
- if string_item.attributes['name'].value == z:
- caf_additions.append(' ' + string_item.toxml())
- found_string.append(z)
- for y in names_cm_string_array:
- if y not in names_base_string_array and y not in found_string_array:
- for string_array_item in list_cm_string_array:
- if string_array_item.attributes['name'].value == y:
- caf_additions.append(' ' + string_array_item.toxml())
- found_string_array.append(y)
- for x in names_cm_plurals:
- if x not in names_base_plurals and x not in found_plurals:
- for plurals_item in list_cm_plurals:
- if plurals_item.attributes['name'].value == x:
- caf_additions.append(' ' + plurals_item.toxml())
- found_plurals.append(x)
-
- # Done :-)
+ # Done
return caf_additions
+def get_default_branch(xml):
+ xml_default = xml.getElementsByTagName('default')[0]
+ xml_default_revision = xml_default.attributes['revision'].value
+ return re.search('refs/heads/(.*)', xml_default_revision).groups()[0]
+
def purge_caf_additions(strings_base, strings_cm):
# Load AOSP file and resources
xml_base = minidom.parse(strings_base)
@@ -213,32 +194,124 @@
file_this.write(addition + '\n')
file_this.close()
-def push_as_commit(path, name, branch):
- # CM gerrit nickname
- username = 'your_nickname'
+def push_as_commit(path, name, branch, username):
+ print('Committing ' + name + ' on branch ' + branch)
# Get path
path = os.getcwd() + '/' + path
- # Create git commit
+ # Create repo object
repo = git.Repo(path)
+
+ # Remove previously deleted files from Git
removed_files = repo.git.ls_files(d=True).split('\n')
try:
repo.git.rm(removed_files)
except:
pass
+
+ # Add all files to commit
repo.git.add('-A')
+
+ # Create commit; if it fails, probably empty so skipping
try:
repo.git.commit(m='Automatic translation import')
except:
print('Failed to create commit for ' + name + ', probably empty: skipping')
return
+
+ # Push commit
repo.git.push('ssh://' + username + '@review.cyanogenmod.org:29418/' + name, 'HEAD:refs/for/' + branch)
+
print('Succesfully pushed commit for ' + name)
+def sync_js_translations(sync_type, path, lang=''):
+ # lang is necessary in download mode
+ if sync_type == 'download' and lang == '':
+ sys.exit('Invalid syntax. Language code is required in download mode.')
+
+ # Read source en.js file. This is necessary for both upload and download modes
+ with codecs.open(path + 'en.js', 'r', 'utf-8') as f:
+ content = f.readlines()
+
+ if sync_type == 'upload':
+ # Prepare XML file structure
+ doc = minidom.Document()
+ header = doc.createElement('resources')
+ file_write = codecs.open(path + 'en.xml', 'w', 'utf-8')
+ else:
+ # Open translation files
+ file_write = codecs.open(path + lang + '.js', 'w', 'utf-8')
+ xml_base = minidom.parse(path + lang + '.xml')
+ tags = xml_base.getElementsByTagName('string')
+
+ # Read each line of en.js
+ for a_line in content:
+ # Regex to determine string id
+ m = re.search(' (.*): [\'|\"]', a_line)
+ if m is not None:
+ for string_id in m.groups():
+ if string_id is not None:
+ # Find string id
+ string_id = string_id.replace(' ', '')
+ m2 = re.search('\'(.*)\'|"(.*)"', a_line)
+ # Find string contents
+ for string_content in m2.groups():
+ if string_content is not None:
+ break
+ if sync_type == 'upload':
+ # In upload mode, create the appropriate string element.
+ contents = doc.createElement('string')
+ contents.attributes['name'] = string_id
+ contents.appendChild(doc.createTextNode(string_content))
+ header.appendChild(contents)
+ else:
+ # In download mode, check if string_id matches a name attribute in the translation XML file.
+ # If it does, replace English text with the translation.
+ # If it does not, English text will remain and will be added to the file to retain the file structure.
+ for string in tags:
+ if string.attributes['name'].value == string_id:
+ a_line = a_line.replace(string_content.rstrip(), string.firstChild.nodeValue)
+ break
+ break
+ # In download mode do not write comments
+ if sync_type == 'download' and not '//' in a_line:
+ # Add language identifier (1)
+ if 'cmaccount.l10n.en' in a_line:
+ a_line = a_line.replace('l10n.en', 'l10n.' + lang)
+ # Add language identifier (2)
+ if 'l10n.add(\'en\'' in a_line:
+ a_line = a_line.replace('l10n.add(\'en\'', 'l10n.add(\'' + lang + '\'')
+ # Now write the line
+ file_write.write(a_line)
+
+ # Create XML file structure
+ if sync_type == 'upload':
+ header.appendChild(contents)
+ contents = header.toxml().replace('<string', '\n <string').replace('</resources>', '\n</resources>')
+ file_write.write('<?xml version="1.0" encoding="utf-8"?>\n')
+ file_write.write('<!-- .JS CONVERTED TO .XML - DO NOT MERGE THIS FILE -->\n')
+ file_write.write(contents)
+
+ # Close file
+ file_write.close()
+
+###################################################################################################
+
print('Welcome to the CM Crowdin sync script!')
-print('\nSTEP 0: Checking dependencies')
+###################################################################################################
+
+parser = argparse.ArgumentParser(description='Synchronising CyanogenMod\'s translations with Crowdin')
+parser.add_argument('--username', help='Gerrit username', required=True)
+#parser.add_argument('--upload-only', help='Only upload CM source translations to Crowdin', required=False)
+args = vars(parser.parse_args())
+
+username = args['username']
+
+############################################## STEP 0 ##############################################
+
+print('\nSTEP 0A: Checking dependencies')
# Check for Ruby version of crowdin-cli
if subprocess.check_output(['rvm', 'all', 'do', 'gem', 'list', 'crowdin-cli', '-i']) == 'true':
sys.exit('You have not installed crowdin-cli. Terminating.')
@@ -281,46 +354,78 @@
else:
print('Found: crowdin/extra_packages.xml')
-print('\nSTEP 1: Remove CAF additions (non-AOSP supported languages)')
-# Load crowdin/caf.xml
-print('Loading crowdin/caf.xml')
+# Check for crowdin/js.xml
+if not os.path.isfile('crowdin/js.xml'):
+ sys.exit('You have no crowdin/js.xml. Terminating.')
+else:
+ print('Found: crowdin/js.xml')
+
+print('\nSTEP 0B: Define shared variables')
+
+# Variables regarding android/default.xml
+print('Loading: android/default.xml')
+xml_android = minidom.parse('android/default.xml')
+
+# Variables regarding crowdin/caf.xml
+print('Loading: crowdin/caf.xml')
xml = minidom.parse('crowdin/caf.xml')
items = xml.getElementsByTagName('item')
+# Variables regarding crowdin/js.xml
+print('Loading: crowdin/js.xml')
+xml_js = minidom.parse('crowdin/js.xml')
+items_js = xml_js.getElementsByTagName('project')
+
+# Default branch
+default_branch = get_default_branch(xml_android)
+print('Default branch: ' + default_branch)
+
+print('\nSTEP 0C: Download AOSP base files')
+for item in items:
+ path_to_values = item.attributes['path'].value
+ subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
+ for aosp_item in item.getElementsByTagName('aosp'):
+ url = aosp_item.firstChild.nodeValue
+ xml_file = aosp_item.attributes['file'].value
+ path_to_base = 'tmp/' + path_to_values + '/' + xml_file
+ urlretrieve(url, path_to_base)
+ print('Downloaded: ' + path_to_base)
+
+############################################## STEP 1 ##############################################
+
+print('\nSTEP 1: Remove CAF additions (non-AOSP supported languages)')
# Store all created cm_caf.xml files in here.
# Easier to remove them afterwards, as they cannot be committed
cm_caf_add = []
for item in items:
# Create tmp dir for download of AOSP base file
- path_to_values = item.attributes["path"].value
- subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
+ path_to_values = item.attributes['path'].value
for aosp_item in item.getElementsByTagName('aosp'):
- url = aosp_item.firstChild.nodeValue
- xml_file = aosp_item.attributes["file"].value
+ xml_file = aosp_item.attributes['file'].value
path_to_base = 'tmp/' + path_to_values + '/' + xml_file
path_to_cm = path_to_values + '/' + xml_file
- urlretrieve(url, path_to_base)
purge_caf_additions(path_to_base, path_to_cm)
cm_caf_add.append(path_to_cm)
print('Purged ' + path_to_cm + ' from CAF additions')
+############################################## STEP 2 ##############################################
+
print('\nSTEP 2: Upload Crowdin source translations (non-AOSP supported languages)')
# Execute 'crowdin-cli upload sources' and show output
print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_aosp.yaml', 'upload', 'sources']))
+############################################## STEP 3 ##############################################
+
print('\nSTEP 3: Revert removal of CAF additions (non-AOSP supported languages)')
for purged_file in cm_caf_add:
os.remove(purged_file)
shutil.move(purged_file + '.backup', purged_file)
print('Reverted purged file ' + purged_file)
-print('\nSTEP 4: Create source cm_caf.xmls (AOSP supported languages)')
-# Load crowdin/caf.xml
-print('Loading crowdin/caf.xml')
-xml = minidom.parse('crowdin/caf.xml')
-items = xml.getElementsByTagName('item')
+############################################## STEP 4 ##############################################
+print('\nSTEP 4: Create source cm_caf.xmls (AOSP supported languages)')
# Store all created cm_caf.xml files in here.
# Easier to remove them afterwards, as they cannot be committed
cm_caf = []
@@ -328,7 +433,6 @@
for item in items:
# Create tmp dir for download of AOSP base file
path_to_values = item.attributes["path"].value
- subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
# Create cm_caf.xml - header
f = codecs.open(path_to_values + '/cm_caf.xml', 'w', 'utf-8')
f.write('<?xml version="1.0" encoding="utf-8"?>\n')
@@ -353,11 +457,9 @@
contents = []
item_aosp = item.getElementsByTagName('aosp')
for aosp_item in item_aosp:
- url = aosp_item.firstChild.nodeValue
xml_file = aosp_item.attributes["file"].value
path_to_base = 'tmp/' + path_to_values + '/' + xml_file
path_to_cm = path_to_values + '/' + xml_file
- urlretrieve(url, path_to_base)
contents = contents + get_caf_additions(path_to_base, path_to_cm)
for addition in contents:
f.write(addition + '\n')
@@ -367,79 +469,129 @@
cm_caf.append(path_to_values + '/cm_caf.xml')
print('Created ' + path_to_values + '/cm_caf.xml')
-print('\nSTEP 5: Upload Crowdin source translations (AOSP supported languages)')
+############################################## STEP 5 ##############################################
+# JS files cannot be translated easily on Crowdin. That's why they are uploaded as Android XML
+# files. At this time, the (English) JS source file is converted to an XML file, so Crowdin knows it
+# needs to download for it.
+print('\nSTEP 5: Convert .js source translations to .xml')
+
+js_files = []
+
+for item in items_js:
+ path = item.attributes['path'].value + '/'
+ sync_js_translations('upload', path)
+ print('Converted: ' + path + 'en.js to en.xml')
+ js_files.append(path + 'en.js')
+
+############################################## STEP 6 ##############################################
+print('\nSTEP 6: Upload Crowdin source translations (AOSP supported languages)')
# Execute 'crowdin-cli upload sources' and show output
print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_cm.yaml', 'upload', 'sources']))
-print('\nSTEP 6A: Download Crowdin translations (AOSP supported languages)')
+############################################## STEP 7 ##############################################
+print('\nSTEP 7A: Download Crowdin translations (AOSP supported languages)')
# Execute 'crowdin-cli download' and show output
print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_cm.yaml', 'download']))
-print('\nSTEP 6B: Download Crowdin translations (non-AOSP supported languages)')
+print('\nSTEP 7B: Download Crowdin translations (non-AOSP supported languages)')
# Execute 'crowdin-cli download' and show output
print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_aosp.yaml', 'download']))
-print('\nSTEP 7: Remove temp dir')
+############################################## STEP 8 ##############################################
+print('\nSTEP 8: Remove temp dir')
# We are done with cm_caf.xml files, so remove tmp/
shutil.rmtree(os.getcwd() + '/tmp')
-print('\nSTEP 8: Remove useless empty translations')
+############################################## STEP 9 ##############################################
+print('\nSTEP 9: Remove useless empty translations')
# Some line of code that I found to find all XML files
result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd()) for f in filenames if os.path.splitext(f)[1] == '.xml']
+empty_contents = {'<resources/>', '<resources xmlns:android="http://schemas.android.com/apk/res/android"/>', '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>', '<resources xmlns:android="http://schemas.android.com/apk/res/android" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>'}
for xml_file in result:
- # We hate empty, useless files. Crowdin exports them with <resources/> (sometimes with xliff).
- # That means: easy to find
- if '<resources/>' in open(xml_file).read():
- print('Removing ' + xml_file)
- os.remove(xml_file)
- elif '<resources xmlns:android="http://schemas.android.com/apk/res/android"/>' in open(xml_file).read():
- print('Removing ' + xml_file)
- os.remove(xml_file)
- elif '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
- print('Removing ' + xml_file)
- os.remove(xml_file)
- elif '<resources xmlns:android="http://schemas.android.com/apk/res/android" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
- print('Removing ' + xml_file)
- os.remove(xml_file)
+ for line in empty_contents:
+ if line in open(xml_file).read():
+ print('Removing ' + xml_file)
+ os.remove(xml_file)
+ break
-print('\nSTEP 9: Create a list of pushable translations')
+for js_file in js_files:
+ print('Removing ' + js_file)
+ os.remove(js_file)
+############################################## STEP 10 #############################################
+print('\nSTEP 10: Create a list of pushable translations')
# Get all files that Crowdin pushed
proc = subprocess.Popen(['crowdin-cli -c crowdin/crowdin_cm.yaml list sources && crowdin-cli -c crowdin/crowdin_aosp.yaml list sources'], stdout=subprocess.PIPE, shell=True)
proc.wait() # Wait for the above to finish
-print('\nSTEP 10: Remove unwanted source cm_caf.xmls (AOSP supported languages)')
+############################################## STEP 11 #############################################
+print('\nSTEP 11: Remove unwanted source cm_caf.xmls (AOSP supported languages)')
# Remove all cm_caf.xml files, which you can find in the list 'cm_caf'
for cm_caf_file in cm_caf:
print('Removing ' + cm_caf_file)
os.remove(cm_caf_file)
-print('\nSTEP 11: Commit to Gerrit')
-xml = minidom.parse('android/default.xml')
+############################################## STEP 12 #############################################
+#print('\nSTEP 12: Convert JS-XML translations to their JS format')
+#
+#for item in items_js:
+# path = item.attributes['path'].value
+# all_xml_files = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd() + '/' + path) for f in filenames if os.path.splitext(f)[1] == '.xml']
+# for xml_file in all_xml_files:
+# lang_code = os.path.splitext(xml_file)[0]
+# sync_js_translations('download', path, lang_code)
+# os.remove(xml_file)
+# os.remove(path + '/' + item.attributes['source'].value)
+#
+############################################## STEP 13 #############################################
+print('\nSTEP 13: Commit to Gerrit')
xml_extra = minidom.parse('crowdin/extra_packages.xml')
-items = xml.getElementsByTagName('project')
+items = xml_android.getElementsByTagName('project')
items += xml_extra.getElementsByTagName('project')
all_projects = []
for path in iter(proc.stdout.readline,''):
# Remove the \n at the end of each line
path = path.rstrip()
- # Get project root dir from Crowdin's output
- m = re.search('/(.*Superuser)/Superuser.*|/(.*LatinIME).*|/(frameworks/base).*|/(.*CMFileManager).*|/(.*CMHome).*|/(device/.*/.*)/.*/res/values.*|/(hardware/.*/.*)/.*/res/values.*|/(.*)/res/values.*', path)
- for good_path in m.groups():
- # When a project has multiple translatable files, Crowdin will give duplicates.
- # We don't want that (useless empty commits), so we save each project in all_projects
- # and check if it's already in there.
- if good_path is not None and not good_path in all_projects:
- all_projects.append(good_path)
- for project_item in items:
- # We need to have the Github repository for the git push url.
- # Obtain them from android/default.xml or crowdin/extra_packages.xml.
- if project_item.attributes['path'].value == good_path:
- if project_item.hasAttribute('revision'):
- branch = project_item.attributes['revision'].value
- else:
- branch = 'cm-11.0'
- print('Committing ' + project_item.attributes['name'].value + ' on branch ' + branch)
- push_as_commit(good_path, project_item.attributes['name'].value, branch)
+ if not path:
+ continue
+
+ # Get project root dir from Crowdin's output by regex
+ m = re.search('/(.*Superuser)/Superuser.*|/(.*LatinIME).*|/(frameworks/base).*|/(.*CMFileManager).*|/(.*CMHome).*|/(device/.*/.*)/.*/res/values.*|/(hardware/.*/.*)/.*/res/values.*|/(.*)/res/values.*', path)
+
+ if not m.groups():
+ # Regex result is empty, warn the user
+ print('WARNING: Cannot determine project root dir of [' + path + '], skipping')
+ continue
+
+ for i in m.groups():
+ if not i:
+ continue
+ result = i
+ break
+
+ if result in all_projects:
+ # Already committed for this project, go to next project
+ continue
+
+ # When a project has multiple translatable files, Crowdin will give duplicates.
+ # We don't want that (useless empty commits), so we save each project in all_projects
+ # and check if it's already in there.
+ all_projects.append(result)
+
+ # Search in android/default.xml or crowdin/extra_packages.xml for the project's name
+ for project_item in items:
+ if project_item.attributes['path'].value != result:
+ # No match found, go to next item
+ continue
+
+ # Define branch (custom branch if defined in xml file, otherwise 'cm-11.0'
+ if project_item.hasAttribute('revision'):
+ branch = project_item.attributes['revision'].value
+ else:
+ branch = default_branch
+
+ push_as_commit(result, project_item.attributes['name'].value, branch, username)
+
+############################################### DONE ###############################################
print('\nDone!')