diff options
Diffstat (limited to 'tools/localize/localize.cpp')
| -rw-r--r-- | tools/localize/localize.cpp | 768 |
1 files changed, 0 insertions, 768 deletions
diff --git a/tools/localize/localize.cpp b/tools/localize/localize.cpp deleted file mode 100644 index 68c03b625cc1..000000000000 --- a/tools/localize/localize.cpp +++ /dev/null @@ -1,768 +0,0 @@ -#include "SourcePos.h" -#include "ValuesFile.h" -#include "XLIFFFile.h" -#include "Perforce.h" -#include "merge_res_and_xliff.h" -#include "localize.h" -#include "file_utils.h" -#include "res_check.h" -#include "xmb.h" - -#include <host/pseudolocalize.h> - -#include <stdlib.h> -#include <stdarg.h> -#include <sstream> -#include <stdio.h> -#include <string.h> -#include <stdlib.h> - -using namespace std; - -FILE* g_logFile = NULL; - -int test(); - -int -read_settings(const string& filename, map<string,Settings>* result, const string& rootDir) -{ - XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); - if (root == NULL) { - SourcePos(filename, -1).Error("Error reading file."); - return 1; - } - - // <configuration> - vector<XMLNode*> configNodes = root->GetElementsByName("", "configuration"); - const size_t I = configNodes.size(); - for (size_t i=0; i<I; i++) { - const XMLNode* configNode = configNodes[i]; - - Settings settings; - settings.id = configNode->GetAttribute("", "id", ""); - if (settings.id == "") { - configNode->Position().Error("<configuration> needs an id attribute."); - delete root; - return 1; - } - - settings.oldVersion = configNode->GetAttribute("", "old-cl", ""); - - settings.currentVersion = configNode->GetAttribute("", "new-cl", ""); - if (settings.currentVersion == "") { - configNode->Position().Error("<configuration> needs a new-cl attribute."); - delete root; - return 1; - } - - // <app> - vector<XMLNode*> appNodes = configNode->GetElementsByName("", "app"); - - const size_t J = appNodes.size(); - for (size_t j=0; j<J; j++) { - const XMLNode* appNode = appNodes[j]; - - string dir = appNode->GetAttribute("", "dir", ""); - if (dir == "") { - appNode->Position().Error("<app> needs a dir attribute."); - delete root; - return 1; - } - - settings.apps.push_back(dir); - } - - // <reject> - vector<XMLNode*> rejectNodes = configNode->GetElementsByName("", "reject"); - - const size_t K = rejectNodes.size(); - for (size_t k=0; k<K; k++) { - const XMLNode* rejectNode = rejectNodes[k]; - - Reject reject; - - reject.file = rejectNode->GetAttribute("", "file", ""); - if (reject.file == "") { - rejectNode->Position().Error("<reject> needs a file attribute."); - delete root; - return 1; - } - string f = reject.file; - reject.file = rootDir; - reject.file += '/'; - reject.file += f; - - reject.name = rejectNode->GetAttribute("", "name", ""); - if (reject.name == "") { - rejectNode->Position().Error("<reject> needs a name attribute."); - delete root; - return 1; - } - - reject.comment = trim_string(rejectNode->CollapseTextContents()); - - settings.reject.push_back(reject); - } - - (*result)[settings.id] = settings; - } - - delete root; - return 0; -} - - -static void -ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename) -{ - const set<StringResource>& strings = values->GetStrings(); - for (set<StringResource>::const_iterator it=strings.begin(); it!=strings.end(); it++) { - StringResource res = *it; - res.file = englishFilename; - xliff->AddStringResource(res); - } -} - -static bool -contains_reject(const Settings& settings, const string& file, const TransUnit& tu) -{ - const string name = tu.id; - const vector<Reject>& reject = settings.reject; - const size_t I = reject.size(); - for (size_t i=0; i<I; i++) { - const Reject& r = reject[i]; - if (r.file == file && r.name == name) { - return true; - } - } - return false; -} - -/** - * If it's been rejected, then we keep whatever info we have. - * - * Implements this truth table: - * - * S AT AS Keep - * ----------------------- - * 0 0 0 0 (this case can't happen) - * 0 0 1 0 (it was there, never translated, and removed) - * 0 1 0 0 (somehow it got translated, but it was removed) - * 0 1 1 0 (it was removed after having been translated) - * - * 1 0 0 1 (it was just added) - * 1 0 1 1 (it was added, has been changed, but it never got translated) - * 1 1 0 1 (somehow it got translated, but we don't know based on what) - * 1 1 1 0/1 (it's in both. 0 if S=AS b/c there's no need to retranslate if they're - * the same. 1 if S!=AS because S changed, so it should be retranslated) - * - * The first four are cases where, whatever happened in the past, the string isn't there - * now, so it shouldn't be in the XLIFF file. - * - * For cases 4 and 5, the string has never been translated, so get it translated. - * - * For case 6, it's unclear where the translated version came from, so we're conservative - * and send it back for them to have another shot at. - * - * For case 7, we have some data. We have two choices. We could rely on the translator's - * translation memory or tools to notice that the strings haven't changed, and populate the - * <target> field themselves. Or if the string hasn't changed since last time, we can just - * not even tell them about it. As the project nears the end, it will be convenient to see - * the xliff files reducing in size, so we pick the latter. Obviously, if the string has - * changed, then we need to get it retranslated. - */ -bool -keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie) -{ - const Settings* settings = reinterpret_cast<const Settings*>(cookie); - - if (contains_reject(*settings, file, unit)) { - return true; - } - - if (unit.source.id == "") { - return false; - } - if (unit.altTarget.id == "" || unit.altSource.id == "") { - return true; - } - return unit.source.value->ContentsToString(XLIFF_NAMESPACES) - != unit.altSource.value->ContentsToString(XLIFF_NAMESPACES); -} - -int -validate_config(const string& settingsFile, const map<string,Settings>& settings, - const string& config) -{ - if (settings.find(config) == settings.end()) { - SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n", - config.c_str()); - return 1; - } - return 0; -} - -int -validate_configs(const string& settingsFile, const map<string,Settings>& settings, - const vector<string>& configs) -{ - int err = 0; - for (size_t i=0; i<configs.size(); i++) { - string config = configs[i]; - err |= validate_config(settingsFile, settings, config); - } - return err; -} - -int -select_files(vector<string> *resFiles, const string& config, - const map<string,Settings>& settings, const string& rootDir) -{ - int err; - vector<vector<string> > allResFiles; - vector<string> configs; - configs.push_back(config); - err = select_files(&allResFiles, configs, settings, rootDir); - if (err == 0) { - *resFiles = allResFiles[0]; - } - return err; -} - -int -select_files(vector<vector<string> > *allResFiles, const vector<string>& configs, - const map<string,Settings>& settings, const string& rootDir) -{ - int err; - printf("Selecting files..."); - fflush(stdout); - - for (size_t i=0; i<configs.size(); i++) { - const string& config = configs[i]; - const Settings& setting = settings.find(config)->second; - - vector<string> resFiles; - err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir, - setting.apps, &resFiles, true); - if (err != 0) { - fprintf(stderr, "error with perforce. bailing\n"); - return err; - } - - allResFiles->push_back(resFiles); - } - return 0; -} - -static int -do_export(const string& settingsFile, const string& rootDir, const string& outDir, - const string& targetLocale, const vector<string>& configs) -{ - bool success = true; - int err; - - if (false) { - printf("settingsFile=%s\n", settingsFile.c_str()); - printf("rootDir=%s\n", rootDir.c_str()); - printf("outDir=%s\n", outDir.c_str()); - for (size_t i=0; i<configs.size(); i++) { - printf("config[%zd]=%s\n", i, configs[i].c_str()); - } - } - - map<string,Settings> settings; - err = read_settings(settingsFile, &settings, rootDir); - if (err != 0) { - return err; - } - - err = validate_configs(settingsFile, settings, configs); - if (err != 0) { - return err; - } - - vector<vector<string> > allResFiles; - err = select_files(&allResFiles, configs, settings, rootDir); - if (err != 0) { - return err; - } - - size_t totalFileCount = 0; - for (size_t i=0; i<allResFiles.size(); i++) { - totalFileCount += allResFiles[i].size(); - } - totalFileCount *= 3; // we try all 3 versions of the file - - size_t fileProgress = 0; - vector<Stats> stats; - vector<pair<string,XLIFFFile*> > xliffs; - - for (size_t i=0; i<configs.size(); i++) { - const string& config = configs[i]; - const Settings& setting = settings[config]; - - if (false) { - fprintf(stderr, "Configuration: %s (%zd of %zd)\n", config.c_str(), i+1, - configs.size()); - fprintf(stderr, " Old CL: %s\n", setting.oldVersion.c_str()); - fprintf(stderr, " Current CL: %s\n", setting.currentVersion.c_str()); - } - - Configuration english; - english.locale = "en_US"; - Configuration translated; - translated.locale = targetLocale; - XLIFFFile* xliff = XLIFFFile::Create(english, translated, setting.currentVersion); - - const vector<string>& resFiles = allResFiles[i]; - const size_t J = resFiles.size(); - for (size_t j=0; j<J; j++) { - string resFile = resFiles[j]; - - // parse the files into a ValuesFile - // pull out the strings and add them to the XLIFFFile - - // current file - print_file_status(++fileProgress, totalFileCount); - ValuesFile* currentFile = get_values_file(resFile, english, CURRENT_VERSION, - setting.currentVersion, true); - if (currentFile != NULL) { - ValuesFile_to_XLIFFFile(currentFile, xliff, resFile); - //printf("currentFile=[%s]\n", currentFile->ToString().c_str()); - } else { - fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(), - setting.currentVersion.c_str()); - success = false; - } - - // old file - print_file_status(++fileProgress, totalFileCount); - ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION, - setting.oldVersion, false); - if (oldFile != NULL) { - ValuesFile_to_XLIFFFile(oldFile, xliff, resFile); - //printf("oldFile=[%s]\n", oldFile->ToString().c_str()); - } - - // translated version - // (get the head of the tree for the most recent translation, but it's considered - // the old one because the "current" one hasn't been made yet, and this goes into - // the <alt-trans> tag if necessary - print_file_status(++fileProgress, totalFileCount); - string transFilename = translated_file_name(resFile, targetLocale); - ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION, - setting.currentVersion, false); - if (transFile != NULL) { - ValuesFile_to_XLIFFFile(transFile, xliff, resFile); - } - - delete currentFile; - delete oldFile; - delete transFile; - } - - Stats beforeFilterStats = xliff->GetStats(config); - - // run through the XLIFFFile and strip out TransUnits that have identical - // old and current source values and are not in the reject list, or just - // old values and no source values - xliff->Filter(keep_this_trans_unit, (void*)&setting); - - Stats afterFilterStats = xliff->GetStats(config); - afterFilterStats.totalStrings = beforeFilterStats.totalStrings; - - // add the reject comments - for (vector<Reject>::const_iterator reject = setting.reject.begin(); - reject != setting.reject.end(); reject++) { - TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name); - tu->rejectComment = reject->comment; - } - - // config-locale-current_cl.xliff - stringstream filename; - if (outDir != "") { - filename << outDir << '/'; - } - filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff"; - xliffs.push_back(pair<string,XLIFFFile*>(filename.str(), xliff)); - - stats.push_back(afterFilterStats); - } - - // today is a good day to die - if (!success || SourcePos::HasErrors()) { - return 1; - } - - // write the XLIFF files - printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s"); - for (vector<pair<string,XLIFFFile*> >::iterator it = xliffs.begin(); it != xliffs.end(); it++) { - const string& filename = it->first; - XLIFFFile* xliff = it->second; - string text = xliff->ToString(); - write_to_file(filename, text); - } - - // the stats - printf("\n" - " to without total\n" - " config files translate comments strings\n" - "-----------------------------------------------------------------------\n"); - Stats totals; - totals.config = "total"; - totals.files = 0; - totals.toBeTranslated = 0; - totals.noComments = 0; - totals.totalStrings = 0; - for (vector<Stats>::iterator it=stats.begin(); it!=stats.end(); it++) { - string cfg = it->config; - if (cfg.length() > 20) { - cfg.resize(20); - } - printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files, - it->toBeTranslated, it->noComments, it->totalStrings); - totals.files += it->files; - totals.toBeTranslated += it->toBeTranslated; - totals.noComments += it->noComments; - totals.totalStrings += it->totalStrings; - } - if (stats.size() > 1) { - printf("-----------------------------------------------------------------------\n" - " %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files, - totals.toBeTranslated, totals.noComments, totals.totalStrings); - } - printf("\n"); - return 0; -} - -struct PseudolocalizeSettings { - XLIFFFile* xliff; - bool expand; -}; - - -string -pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings) -{ - return pseudolocalize_string(source); -} - -static XMLNode* -pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings) -{ - if (source->Type() == XMLNode::TEXT) { - return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings), - source->Pretty()); - } else { - XMLNode* target; - if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") { - // XXX don't translate these - target = XMLNode::NewElement(source->Position(), source->Namespace(), - source->Name(), source->Attributes(), source->Pretty()); - } else { - target = XMLNode::NewElement(source->Position(), source->Namespace(), - source->Name(), source->Attributes(), source->Pretty()); - } - - const vector<XMLNode*>& children = source->Children(); - const size_t I = children.size(); - for (size_t i=0; i<I; i++) { - target->EditChildren().push_back(pseudolocalize_xml_node(children[i], settings)); - } - - return target; - } -} - -void -pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie) -{ - const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie; - - const StringResource& source = unit->source; - StringResource* target = &unit->target; - *target = source; - - target->config = settings->xliff->TargetConfig(); - - delete target->value; - target->value = pseudolocalize_xml_node(source.value, settings); -} - -int -pseudolocalize_xliff(XLIFFFile* xliff, bool expand) -{ - PseudolocalizeSettings settings; - - settings.xliff = xliff; - settings.expand = expand; - xliff->Map(pseudolocalize_trans_unit, &settings); - return 0; -} - -static int -do_pseudo(const string& infile, const string& outfile, bool expand) -{ - int err; - - XLIFFFile* xliff = XLIFFFile::Parse(infile); - if (xliff == NULL) { - return 1; - } - - pseudolocalize_xliff(xliff, expand); - - err = write_to_file(outfile, xliff->ToString()); - - delete xliff; - - return err; -} - -void -log_printf(const char *fmt, ...) -{ - int ret; - va_list ap; - - if (g_logFile != NULL) { - va_start(ap, fmt); - ret = vfprintf(g_logFile, fmt, ap); - va_end(ap); - fflush(g_logFile); - } -} - -void -close_log_file() -{ - if (g_logFile != NULL) { - fclose(g_logFile); - } -} - -void -open_log_file(const char* file) -{ - g_logFile = fopen(file, "w"); - printf("log file: %s -- %p\n", file, g_logFile); - atexit(close_log_file); -} - -static int -usage() -{ - fprintf(stderr, - "usage: localize export OPTIONS CONFIGS...\n" - " REQUIRED OPTIONS\n" - " --settings SETTINGS The settings file to use. See CONFIGS below.\n" - " --root TREE_ROOT The location in Perforce of the files. e.g. //device\n" - " --target LOCALE The target locale. See LOCALES below.\n" - "\n" - " OPTIONAL OPTIONS\n" - " --out DIR Directory to put the output files. Defaults to the\n" - " current directory if not supplied. Files are\n" - " named as follows:\n" - " CONFIG-LOCALE-CURRENT_CL.xliff\n" - "\n" - "\n" - "usage: localize import XLIFF_FILE...\n" - "\n" - "Import a translated XLIFF file back into the tree.\n" - "\n" - "\n" - "usage: localize xlb XMB_FILE VALUES_FILES...\n" - "\n" - "Read resource files from the tree file and write the corresponding XLB file\n" - "\n" - "Supply all of the android resource files (values files) to export after that.\n" - "\n" - "\n" - "\n" - "CONFIGS\n" - "\n" - "LOCALES\n" - "Locales are specified in the form en_US They will be processed correctly\n" - "to locate the resouce files in the tree.\n" - "\n" - "\n" - "usage: localize pseudo OPTIONS INFILE [OUTFILE]\n" - " OPTIONAL OPTIONS\n" - " --big Pad strings so they get longer.\n" - "\n" - "Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n" - "OUTFILE is specified, the results are written there; otherwise, the results are\n" - "written back to INFILE.\n" - "\n" - "\n" - "usage: localize rescheck FILES...\n" - "\n" - "Reads the base strings and prints warnings about bad resources from the given files.\n" - "\n"); - return 1; -} - -int -main(int argc, const char** argv) -{ - //open_log_file("log.txt"); - //g_logFile = stdout; - - if (argc == 2 && 0 == strcmp(argv[1], "--test")) { - return test(); - } - - if (argc < 2) { - return usage(); - } - - int index = 1; - - if (0 == strcmp("export", argv[index])) { - string settingsFile; - string rootDir; - string outDir; - string baseLocale = "en"; - string targetLocale; - string language, region; - vector<string> configs; - - index++; - while (index < argc) { - if (0 == strcmp("--settings", argv[index])) { - settingsFile = argv[index+1]; - index += 2; - } - else if (0 == strcmp("--root", argv[index])) { - rootDir = argv[index+1]; - index += 2; - } - else if (0 == strcmp("--out", argv[index])) { - outDir = argv[index+1]; - index += 2; - } - else if (0 == strcmp("--target", argv[index])) { - targetLocale = argv[index+1]; - index += 2; - } - else if (argv[index][0] == '-') { - fprintf(stderr, "unknown argument %s\n", argv[index]); - return usage(); - } - else { - break; - } - } - for (; index<argc; index++) { - configs.push_back(argv[index]); - } - - if (settingsFile == "" || rootDir == "" || configs.size() == 0 || targetLocale == "") { - return usage(); - } - if (!split_locale(targetLocale, &language, ®ion)) { - fprintf(stderr, "illegal --target locale: '%s'\n", targetLocale.c_str()); - return usage(); - } - - - return do_export(settingsFile, rootDir, outDir, targetLocale, configs); - } - else if (0 == strcmp("import", argv[index])) { - vector<string> xliffFilenames; - - index++; - for (; index<argc; index++) { - xliffFilenames.push_back(argv[index]); - } - - return do_merge(xliffFilenames); - } - else if (0 == strcmp("xlb", argv[index])) { - string outfile; - vector<string> resFiles; - - index++; - if (argc < index+1) { - return usage(); - } - - outfile = argv[index]; - - index++; - for (; index<argc; index++) { - resFiles.push_back(argv[index]); - } - - return do_xlb_export(outfile, resFiles); - } - else if (0 == strcmp("pseudo", argv[index])) { - string infile; - string outfile; - bool big = false; - - index++; - while (index < argc) { - if (0 == strcmp("--big", argv[index])) { - big = true; - index += 1; - } - else if (argv[index][0] == '-') { - fprintf(stderr, "unknown argument %s\n", argv[index]); - return usage(); - } - else { - break; - } - } - - if (index == argc-1) { - infile = argv[index]; - outfile = argv[index]; - } - else if (index == argc-2) { - infile = argv[index]; - outfile = argv[index+1]; - } - else { - fprintf(stderr, "unknown argument %s\n", argv[index]); - return usage(); - } - - return do_pseudo(infile, outfile, big); - } - else if (0 == strcmp("rescheck", argv[index])) { - vector<string> files; - - index++; - while (index < argc) { - if (argv[index][0] == '-') { - fprintf(stderr, "unknown argument %s\n", argv[index]); - return usage(); - } - else { - break; - } - } - for (; index<argc; index++) { - files.push_back(argv[index]); - } - - if (files.size() == 0) { - return usage(); - } - - return do_rescheck(files); - } - else { - return usage(); - } - - if (SourcePos::HasErrors()) { - SourcePos::PrintErrors(stderr); - return 1; - } - - return 0; -} - |