Numerous fixes
- Add support for multiple base files
- Add comments
- Small fixes and improvements
diff --git a/README.mkdn b/README.mkdn
index dee3ac3..d31b672 100644
--- a/README.mkdn
+++ b/README.mkdn
@@ -32,4 +32,5 @@
Bugs
----
- - cm_caf.xml can only be generated once.
+ - When committing fails, the reason of it cannot be determined. Often this is just when there
+ are no new translations, so the script will not exit when this happens.
diff --git a/caf.xml b/caf.xml
index 21a0119..b9361ff 100644
--- a/caf.xml
+++ b/caf.xml
@@ -1,24 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
- Copyright (C) 2014 The CyanogenMod Project (Marco Brohet)
+ Copyright (C) 2014 The CyanogenMod Project
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License
- as published by the Free Software Foundation; either version 2
- of the License, or (at your option) any later version.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
+ http://www.apache.org/licenses/LICENSE-2.0
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- MA 02110-1301, USA.
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
-->
<files>
<item path="packages/apps/Settings/res/values">
- <aosp file="strings.xml">https://raw.github.com/android/platform_packages_apps_settings/android-4.4_r1.2/res/values/strings.xml</aosp>
+ <aosp file="strings.xml">https://raw.github.com/android/platform_packages_apps_settings/android-4.4.2_r1/res/values/strings.xml</aosp>
</item>
</files>
diff --git a/cm_crowdin_sync.py b/cm_crowdin_sync.py
index ddffd5b..b40cceb 100755
--- a/cm_crowdin_sync.py
+++ b/cm_crowdin_sync.py
@@ -3,9 +3,22 @@
# cm_crowdin_sync.py
#
# Updates Crowdin source translations and pulls translations
-# directly to CyanogenMod's Git
+# directly to CyanogenMod's Git.
+#
+# Copyright (C) 2014 The CyanogenMod Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-import create_cm_caf_xml
import git
import mmap
import os
@@ -17,52 +30,158 @@
from urllib import urlretrieve
from xml.dom import minidom
-print('Welcome to the CM Crowdin sync script!\n')
+def get_caf_additions(strings_base, strings_cm):
+ # Load AOSP file and resources
+ xml_base = minidom.parse(strings_base)
+ list_base_string = xml_base.getElementsByTagName('string')
+ list_base_string_array = xml_base.getElementsByTagName('string-array')
+ list_base_plurals = xml_base.getElementsByTagName('plurals')
+ # Load CM file and resources
+ xml_cm = minidom.parse(strings_cm)
+ list_cm_string = xml_cm.getElementsByTagName('string')
+ list_cm_string_array = xml_cm.getElementsByTagName('string-array')
+ list_cm_plurals = xml_cm.getElementsByTagName('plurals')
-print('STEP 0: Checking dependencies\n')
+ # All names from CM
+ names_cm_string = []
+ names_cm_string_array = []
+ names_cm_plurals = []
+ # All names from AOSP
+ names_base_string = []
+ names_base_string_array = []
+ names_base_plurals = []
+
+ # Get all names from CM
+ for s in list_cm_string :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_string.append(s.attributes['name'].value)
+ for s in list_cm_string_array :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_string_array.append(s.attributes['name'].value)
+ for s in list_cm_plurals :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_plurals.append(s.attributes['name'].value)
+ # Get all names from AOSP
+ for s in list_base_string :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_string.append(s.attributes['name'].value)
+ for s in list_base_string_array :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_string_array.append(s.attributes['name'].value)
+ for s in list_base_plurals :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_plurals.append(s.attributes['name'].value)
+
+ # Store all differences in this list
+ caf_additions = []
+
+ # Add all CAF additions to the list 'caf_additions'
+ for z in names_cm_string:
+ if not z in names_base_string:
+ caf_additions.append(' ' + list_cm_string[names_cm_string.index(z)].toxml())
+ for z in names_cm_string_array:
+ if not z in names_base_string_array:
+ caf_additions.append(' ' + list_cm_string_array[names_cm_string_array.index(z)].toxml())
+ for z in names_cm_plurals:
+ if not z in names_base_plurals:
+ caf_additions.append(' ' + list_cm_plurals[names_cm_plurals.index(z)].toxml())
+
+ # Done :-)
+ return caf_additions
+
+def push_as_commit(path, name):
+ # Get path
+ path = os.getcwd() + '/' + path
+
+ # Create git commit
+ repo = git.Repo(path)
+ repo.git.add(path)
+ try:
+ repo.git.commit(m='DO NOT MERGE: Automatic translation import test commit')
+# repo.git.push('ssh://cobjeM@review.cyanogenmod.org:29418/' + name, 'HEAD:refs/for/cm-11.0')
+ print 'Succesfully pushed commit for ' + name
+ except:
+ # If git commit fails, it's probably because of no changes.
+ # Just continue.
+ print 'No commit pushed (probably empty?) for ' + name
+ print 'WARNING: If the repository name was not obtained from default.xml, the name might be wrong!'
+
+print('Welcome to the CM Crowdin sync script!')
+
+print('\nSTEP 0: Checking dependencies')
if subprocess.check_output(['rvm', 'all', 'do', 'gem', 'list', 'crowdin-cli', '-i']) == 'true':
sys.exit('You have not installed crowdin-cli. Terminating.')
+else:
+ print('Found: crowdin-cli')
if not os.path.isfile('caf.xml'):
sys.exit('You have no caf.xml. Terminating.')
+else:
+ print('Found: caf.xml')
if not os.path.isfile('default.xml'):
sys.exit('You have no default.xml. Terminating.')
+else:
+ print('Found: default.xml')
-print('STEP 1: Create cm_caf.xml')
+print('\nSTEP 1: Create cm_caf.xml')
+# Load caf.xml
xml = minidom.parse('caf.xml')
items = xml.getElementsByTagName('item')
+# Store all created cm_caf.xml files in here.
+# Easier to remove them afterwards, as they cannot be committed
cm_caf = []
for item in items:
- subprocess.call(['mkdir', '-p', 'tmp/' + item.attributes["path"].value])
+ # Create tmp dir for download of AOSP base file
+ path_to_values = item.attributes["path"].value
+ subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
+ # Create cm_caf.xml - header
+ f = open(path_to_values + '/cm_caf.xml','w')
+ f.write('<?xml version="1.0" encoding="utf-8"?>\n')
+ f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
+ # Create cm_caf.xml - contents
+ # This means we also support multiple base files (e.g. checking if strings.xml and arrays.xml are changed)
+ contents = []
item_aosp = item.getElementsByTagName('aosp')
for aosp_item in item_aosp:
url = aosp_item.firstChild.nodeValue
- path_to_base = 'tmp/' + item.attributes["path"].value + '/' + aosp_item.attributes["file"].value
- path_to_cm = item.attributes["path"].value + '/' + aosp_item.attributes["file"].value
- path = item.attributes["path"].value
+ xml_file = aosp_item.attributes["file"].value
+ path_to_base = 'tmp/' + path_to_values + '/' + xml_file
+ path_to_cm = path_to_values + '/' + xml_file
urlretrieve(url, path_to_base)
- create_cm_caf_xml.create_cm_caf_xml(path_to_base, path_to_cm, path)
- cm_caf.append(path + '/cm_caf.xml')
- print('Created ' + path + '/cm_caf.xml')
+ contents = contents + get_caf_additions(path_to_base, path_to_cm)
+ for addition in contents:
+ f.write(addition + '\n')
+ # Create cm_caf.xml - the end
+ f.write('</resources>')
+ f.close()
+ cm_caf.append(path_to_values + '/cm_caf.xml')
+ print('Created ' + path_to_values + '/cm_caf.xml')
print('\nSTEP 2: Upload Crowdin source translations')
+# Execute 'crowdin-cli upload sources' and show output
print(subprocess.check_output(['crowdin-cli', 'upload', 'sources']))
print('STEP 3: Download Crowdin translations')
+# Execute 'crowdin-cli download' and show output
print(subprocess.check_output(['crowdin-cli', "download"]))
print('STEP 4A: Clean up of source cm_caf.xmls')
+# Remove all cm_caf.xml files, which you can find in the list 'cm_caf'
for cm_caf_file in cm_caf:
print ('Removing ' + cm_caf_file)
os.remove(cm_caf_file)
print('\nSTEP 4B: Clean up of temp dir')
+# We are done with cm_caf.xml files, so remove tmp/
shutil.rmtree(os.getcwd() + '/tmp')
-print('STEP 4C: Clean up of empty translations')
+print('\nSTEP 4C: Clean up of empty translations')
+# Some line of code that I found to find all XML files
result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd()) for f in filenames if os.path.splitext(f)[1] == '.xml']
for xml_file in result:
+ # We hate empty, useless files. Crowdin exports them with <resources/> (sometimes with xliff).
+ # That means: easy to find
if '<resources/>' in open(xml_file).read():
print ('Removing ' + xml_file)
os.remove(xml_file)
@@ -71,25 +190,36 @@
os.remove(xml_file)
print('\nSTEP 5: Push translations to Git')
+# Get all files that Crowdin pushed
proc = subprocess.Popen(['crowdin-cli', 'list', 'sources'],stdout=subprocess.PIPE)
xml = minidom.parse('default.xml')
items = xml.getElementsByTagName('project')
all_projects = []
for path in iter(proc.stdout.readline,''):
+ # Remove the \n at the end of each line
path = path.rstrip()
+ # Get project root dir from Crowdin's output
m = re.search('/(.*Superuser)/Superuser.*|/(.*LatinIME).*|/(frameworks/base).*|/(.*CMFileManager).*|/(device/.*/.*)/.*/res/values.*|/(hardware/.*/.*)/.*/res/values.*|/(.*)/res/values.*', path)
for good_path in m.groups():
+ # When a project has multiple translatable files, Crowdin will give duplicates.
+ # We don't want that (useless empty commits), so we save each project in all_projects
+ # and check if it's already in there.
if good_path is not None and not good_path in all_projects:
all_projects.append(good_path)
working = 'false'
for project_item in items:
+ # We need to have the Github repository for the git push url. Obtain them from
+ # default.xml based on the project root dir.
if project_item.attributes["path"].value == good_path:
working = 'true'
- create_cm_caf_xml.push_as_commit(good_path, project_item.attributes['name'].value)
- print 'WORKING: ' + project_item.attributes['name'].value
+ push_as_commit(good_path, project_item.attributes['name'].value)
+ print 'Committing ' + project_item.attributes['name'].value + ' (based on default.xml)'
+ # We also translate repositories that are not downloaded by default (e.g. device parts).
+ # This is just a fallback.
+ # WARNING: If the name is wrong, this will not stop the script.
if working == 'false':
- create_cm_caf_xml.push_as_commit(good_path, 'CyanogenMod/android_' + good_path.replace('/', '_'))
- print 'WORKAROUND: ' + good_path
+ push_as_commit(good_path, 'CyanogenMod/android_' + good_path.replace('/', '_'))
+ print 'Committing ' + project_item.attributes['name'].value + ' (workaround)'
print('STEP 6: Done!')
diff --git a/create_cm_caf_xml.py b/create_cm_caf_xml.py
deleted file mode 100755
index 6232114..0000000
--- a/create_cm_caf_xml.py
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/python2
-#
-# create_cm_caf_xml.py
-#
-# Generates cm_caf.xml based on the strings that were added to
-# CM's *.xml compared to AOSP's *.xml
-
-import git
-import os
-from xml.dom import minidom
-
-def create_cm_caf_xml(strings_base, strings_cm, path):
- # Load AOSP file and resources
- xml_base = minidom.parse(strings_base)
- list_base_string = xml_base.getElementsByTagName('string')
- list_base_string_array = xml_base.getElementsByTagName('string-array')
- list_base_plurals = xml_base.getElementsByTagName('plurals')
-
- # Load CM file and resources
- xml_cm = minidom.parse(strings_cm)
- list_cm_string = xml_cm.getElementsByTagName('string')
- list_cm_string_array = xml_cm.getElementsByTagName('string-array')
- list_cm_plurals = xml_cm.getElementsByTagName('plurals')
-
- # All names from CM
- names_cm_string = []
- names_cm_string_array = []
- names_cm_plurals = []
- # All names from AOSP
- names_base_string = []
- names_base_string_array = []
- names_base_plurals = []
-
- # Get all names from CM
- for s in list_cm_string :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_string.append(s.attributes['name'].value)
- for s in list_cm_string_array :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_string-array.append(s.attributes['name'].value)
- for s in list_cm_plurals :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_cm_plurals.append(s.attributes['name'].value)
-
- # Get all names from AOSP
- for s in list_base_string :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_string.append(s.attributes['name'].value)
- for s in list_base_string_array :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_string_array.append(s.attributes['name'].value)
- for s in list_base_plurals :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_plurals.append(s.attributes['name'].value)
-
- # Search for different names. If so, output the full line
- t = 5
-
- for z in names_cm_string:
- if not z in names_base_string:
- if t == 5:
- f = open(path + '/cm_caf.xml','w')
- f.write('<?xml version="1.0" encoding="utf-8"?>\n')
- f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
- t = 4
- f.write(list_cm_string[names_cm_string.index(z)].toxml() + '\n')
- for z in names_cm_string_array:
- if not z in names_base_string_array:
- if t == 5:
- f = open(path + '/cm_caf.xml','w')
- f.write('<?xml version="1.0" encoding="utf-8"?>\n')
- f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
- t = 4
- f.write(list_cm_string_array[names_cm_string_array.index(z)].toxml() + '\n')
- for z in names_cm_plurals:
- if not z in names_base_plurals:
- if t == 5:
- f = open(path + '/cm_caf.xml','w')
- f.write('<?xml version="1.0" encoding="utf-8"?>\n')
- f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
- t = 4
- f.write(list_cm_plurals[names_cm_plurals.index(z)].toxml() + '\n')
-
- if t == 4:
- f.write('</resources>')
- f.close()
-
-def push_as_commit(path, name):
- path = os.getcwd() + '/' + path
- repo = git.Repo(path)
- repo.git.add(path)
- try:
- repo.git.commit(m='Crowdin translation import')
- repo.git.push('ssh://cobjeM@review.cyanogenmod.org:29418/' + name, 'HEAD:refs/for/cm-11.0')
- print 'Succesfully pushed commit for' + name
- except:
- print 'No commit pushed (probably empty?) for ' + name
diff --git a/crowdin.yaml b/crowdin.yaml
index 481a73e..345d59c 100644
--- a/crowdin.yaml
+++ b/crowdin.yaml
@@ -1,3 +1,21 @@
+# crowdin.yaml
+#
+# Crowdin configuration file for CyanogenMod
+#
+# Copyright (C) 2014 The CyanogenMod Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
project_identifier: cyanogenmod
api_key: your_api_key
base_path: your/path