diff options
| -rw-r--r-- | Android.bp | 4 | ||||
| -rw-r--r-- | proguard.flags | 7 | ||||
| -rw-r--r-- | src/com/android/providers/media/MediaProvider.java | 37 | ||||
| -rw-r--r-- | src/com/android/providers/media/MediaScannerService.java | 2 | ||||
| -rw-r--r-- | src/com/android/providers/media/MediaService.java | 28 | ||||
| -rw-r--r-- | src/com/android/providers/media/scan/LegacyMediaScanner.java | 69 | ||||
| -rw-r--r-- | src/com/android/providers/media/scan/MediaScanner.java | 32 | ||||
| -rw-r--r-- | src/com/android/providers/media/scan/ModernMediaScanner.java | 585 | ||||
| -rw-r--r-- | tests/res/raw/test_audio.mp3 | bin | 0 -> 18429 bytes | |||
| -rw-r--r-- | tests/res/raw/test_image.jpg | bin | 0 -> 304645 bytes | |||
| -rw-r--r-- | tests/res/raw/test_video.mp4 | bin | 0 -> 135632 bytes | |||
| -rw-r--r-- | tests/src/com/android/providers/media/MediaScannerTest.java | 249 |
12 files changed, 970 insertions, 43 deletions
diff --git a/Android.bp b/Android.bp index 15e25257e..c71dc5eb9 100644 --- a/Android.bp +++ b/Android.bp @@ -12,6 +12,10 @@ android_app { "src/**/*.java", ], + optimize: { + proguard_flags_files: ["proguard.flags"], + }, + platform_apis: true, certificate: "media", diff --git a/proguard.flags b/proguard.flags new file mode 100644 index 000000000..b07d602ac --- /dev/null +++ b/proguard.flags @@ -0,0 +1,7 @@ +-keep class com.android.providers.media.scan.MediaScanner { + *; +} + +-keep class * implements com.android.providers.media.scan.MediaScanner { + *; +} diff --git a/src/com/android/providers/media/MediaProvider.java b/src/com/android/providers/media/MediaProvider.java index a5ad96c74..dd45c9981 100644 --- a/src/com/android/providers/media/MediaProvider.java +++ b/src/com/android/providers/media/MediaProvider.java @@ -72,7 +72,6 @@ import android.graphics.BitmapFactory; import android.graphics.drawable.Icon; import android.media.ExifInterface; import android.media.MediaFile; -import android.media.MediaScanner; import android.media.MediaScannerConnection; import android.media.MediaScannerConnection.MediaScannerConnectionClient; import android.media.ThumbnailUtils; @@ -125,6 +124,7 @@ import com.android.internal.annotations.VisibleForTesting; import com.android.internal.os.BackgroundThread; import com.android.internal.util.ArrayUtils; import com.android.internal.util.IndentingPrintWriter; +import com.android.providers.media.scan.MediaScanner; import libcore.io.IoUtils; import libcore.net.MimeUtils; @@ -2131,7 +2131,7 @@ public class MediaProvider extends ContentProvider { // to use that exact type, so don't override it based on mimetype if (!values.containsKey(FileColumns.MEDIA_TYPE) && mediaType == FileColumns.MEDIA_TYPE_NONE && - !MediaScanner.isNoMediaPath(path)) { + !android.media.MediaScanner.isNoMediaPath(path)) { if (MediaFile.isAudioMimeType(mimeType)) { mediaType = FileColumns.MEDIA_TYPE_AUDIO; } else if (MediaFile.isVideoMimeType(mimeType)) { @@ -2764,7 +2764,7 @@ public class MediaProvider extends ContentProvider { private void hidePath(String volumeName, DatabaseHelper helper, SQLiteDatabase db, String path) { // a new nomedia path was added, so clear the media paths - MediaScanner.clearMediaPathCache(true /* media */, false /* nomedia */); + android.media.MediaScanner.clearMediaPathCache(true /* media */, false /* nomedia */); File nomedia = new File(path); String hiddenroot = nomedia.isDirectory() ? path : nomedia.getParent(); @@ -2806,7 +2806,7 @@ public class MediaProvider extends ContentProvider { */ private void processRemovedNoMediaPath(final String path) { // a nomedia path was removed, so clear the nomedia paths - MediaScanner.clearMediaPathCache(false /* media */, true /* nomedia */); + android.media.MediaScanner.clearMediaPathCache(false /* media */, true /* nomedia */); final String volumeName = MediaStore.getVolumeName(new File(path)); final Uri uri = MediaStore.Files.getContentUri(volumeName); @@ -3423,11 +3423,12 @@ public class MediaProvider extends ContentProvider { if (INTERNAL_VOLUME.equals(mMediaScannerVolume)) { // persist current build fingerprint as fingerprint for system (internal) sound scan - final SharedPreferences scanSettings = - getContext().getSharedPreferences(MediaScanner.SCANNED_BUILD_PREFS_NAME, - Context.MODE_PRIVATE); + final SharedPreferences scanSettings = getContext().getSharedPreferences( + android.media.MediaScanner.SCANNED_BUILD_PREFS_NAME, + Context.MODE_PRIVATE); final SharedPreferences.Editor editor = scanSettings.edit(); - editor.putString(MediaScanner.LAST_INTERNAL_SCAN_FINGERPRINT, Build.FINGERPRINT); + editor.putString(android.media.MediaScanner.LAST_INTERNAL_SCAN_FINGERPRINT, + Build.FINGERPRINT); editor.apply(); } mMediaScannerVolume = null; @@ -3657,12 +3658,8 @@ public class MediaProvider extends ContentProvider { final Bundle res = new Bundle(); switch (method) { case MediaStore.SCAN_FILE_CALL: - final String path = systemFile.getAbsolutePath(); - final String ext = path.substring(path.lastIndexOf('.') + 1); - final String mimeType = MimeUtils.guessMimeTypeFromExtension(ext); res.putParcelable(Intent.EXTRA_STREAM, - MediaService.onScanFile(getContext(), Uri.fromFile(systemFile), - mimeType)); + MediaScanner.instance(getContext()).scanFile(systemFile)); break; case MediaStore.SCAN_VOLUME_CALL: MediaService.onScanVolume(getContext(), Uri.fromFile(systemFile)); @@ -4385,11 +4382,7 @@ public class MediaProvider extends ContentProvider { try (Cursor c = queryForSingleItem(uri, new String[] { FileColumns.DATA }, null, null, null)) { final String data = c.getString(0); - try (MediaScanner scanner = new MediaScanner(getContext(), volumeName)) { - final String ext = data.substring(data.lastIndexOf('.') + 1); - scanner.scanSingleFile(data, - MimeUtils.guessMimeTypeFromExtension(ext)); - } + MediaScanner.instance(getContext()).scanFile(new File(data)); } catch (Exception e) { Log.w(TAG, "Failed to update metadata for " + uri, e); } finally { @@ -4715,13 +4708,7 @@ public class MediaProvider extends ContentProvider { update(uri, values, null, null); break; default: - final String volumeName = MediaStore.getVolumeName(uri); - final String data = file.getAbsolutePath(); - try (MediaScanner scanner = new MediaScanner(getContext(), volumeName)) { - final String ext = data.substring(data.lastIndexOf('.') + 1); - scanner.scanSingleFile(data, - MimeUtils.guessMimeTypeFromExtension(ext)); - } + MediaScanner.instance(getContext()).scanFile(file); break; } } catch (Exception e2) { diff --git a/src/com/android/providers/media/MediaScannerService.java b/src/com/android/providers/media/MediaScannerService.java index a7b6ff011..d1635980b 100644 --- a/src/com/android/providers/media/MediaScannerService.java +++ b/src/com/android/providers/media/MediaScannerService.java @@ -50,7 +50,7 @@ public class MediaScannerService extends Service { .translateAppToSystem(new File(path).getCanonicalFile(), callingPid, callingUid); res = MediaService.onScanFile(MediaScannerService.this, - Uri.fromFile(systemFile), mimeType); + Uri.fromFile(systemFile)); Log.d(TAG, "Scanned " + path + " as " + systemFile + " for " + res); } catch (Exception e) { Log.w(TAG, "Failed to scan " + path, e); diff --git a/src/com/android/providers/media/MediaService.java b/src/com/android/providers/media/MediaService.java index dab5c492d..b752204cb 100644 --- a/src/com/android/providers/media/MediaService.java +++ b/src/com/android/providers/media/MediaService.java @@ -24,7 +24,6 @@ import android.content.ContentResolver; import android.content.ContentValues; import android.content.Context; import android.content.Intent; -import android.media.MediaScanner; import android.net.Uri; import android.os.Environment; import android.os.PowerManager; @@ -32,10 +31,12 @@ import android.os.Trace; import android.provider.MediaStore; import android.util.Log; +import com.android.providers.media.scan.MediaScanner; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collection; public class MediaService extends IntentService { public MediaService() { @@ -77,7 +78,7 @@ public class MediaService extends IntentService { break; } case Intent.ACTION_MEDIA_SCANNER_SCAN_FILE: { - onScanFile(this, intent.getData(), intent.getType()); + onScanFile(this, intent.getData()); break; } default: { @@ -137,8 +138,8 @@ public class MediaService extends IntentService { context.sendBroadcast(new Intent(Intent.ACTION_MEDIA_SCANNER_STARTED, uri)); } - try (MediaScanner scanner = new MediaScanner(context, volumeName)) { - scanner.scanDirectories(resolveDirectories(volumeName)); + for (File dir : resolveDirectories(volumeName)) { + MediaScanner.instance(context).scanDirectory(dir); } resolver.delete(scanUri, null, null); @@ -150,20 +151,13 @@ public class MediaService extends IntentService { } } - public static Uri onScanFile(Context context, Uri uri, String mimeType) throws IOException { + public static Uri onScanFile(Context context, Uri uri) throws IOException { final File file = new File(uri.getPath()).getCanonicalFile(); - final String volumeName = MediaStore.getVolumeName(file); - - try (MediaScanner scanner = new MediaScanner(context, volumeName)) { - return scanner.scanSingleFile(file.getAbsolutePath(), mimeType); - } + return MediaScanner.instance(context).scanFile(file); } - private static String[] resolveDirectories(String volumeName) throws FileNotFoundException { - final ArrayList<String> res = new ArrayList<>(); - for (File dir : MediaStore.getVolumeScanPaths(volumeName)) { - res.add(dir.getAbsolutePath()); - } - return res.toArray(new String[res.size()]); + private static Collection<File> resolveDirectories(String volumeName) + throws FileNotFoundException { + return MediaStore.getVolumeScanPaths(volumeName); } } diff --git a/src/com/android/providers/media/scan/LegacyMediaScanner.java b/src/com/android/providers/media/scan/LegacyMediaScanner.java new file mode 100644 index 000000000..8e3a55760 --- /dev/null +++ b/src/com/android/providers/media/scan/LegacyMediaScanner.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.providers.media.scan; + +import android.content.Context; +import android.net.Uri; +import android.os.Trace; +import android.provider.MediaStore; + +import libcore.net.MimeUtils; + +import java.io.File; + +public class LegacyMediaScanner implements MediaScanner { + private final Context mContext; + + public LegacyMediaScanner(Context context) { + mContext = context; + } + + @Override + public Context getContext() { + return mContext; + } + + @Override + public void scanDirectory(File file) { + final String path = file.getAbsolutePath(); + final String volumeName = MediaStore.getVolumeName(file); + + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "scanDirectory"); + try (android.media.MediaScanner scanner = + new android.media.MediaScanner(mContext, volumeName)) { + scanner.scanDirectories(new String[] { path }); + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + } + + @Override + public Uri scanFile(File file) { + final String path = file.getAbsolutePath(); + final String volumeName = MediaStore.getVolumeName(file); + + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "scanFile"); + try (android.media.MediaScanner scanner = + new android.media.MediaScanner(mContext, volumeName)) { + final String ext = path.substring(path.lastIndexOf('.') + 1); + return scanner.scanSingleFile(path, + MimeUtils.guessMimeTypeFromExtension(ext)); + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + } +} diff --git a/src/com/android/providers/media/scan/MediaScanner.java b/src/com/android/providers/media/scan/MediaScanner.java new file mode 100644 index 000000000..556d522b5 --- /dev/null +++ b/src/com/android/providers/media/scan/MediaScanner.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.providers.media.scan; + +import android.content.Context; +import android.net.Uri; + +import java.io.File; + +public interface MediaScanner { + public Context getContext(); + public void scanDirectory(File file); + public Uri scanFile(File file); + + public static MediaScanner instance(Context context) { + return new LegacyMediaScanner(context); + } +} diff --git a/src/com/android/providers/media/scan/ModernMediaScanner.java b/src/com/android/providers/media/scan/ModernMediaScanner.java new file mode 100644 index 000000000..b07a97d5e --- /dev/null +++ b/src/com/android/providers/media/scan/ModernMediaScanner.java @@ -0,0 +1,585 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.providers.media.scan; + +import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUM; +import static android.media.MediaMetadataRetriever.METADATA_KEY_ALBUMARTIST; +import static android.media.MediaMetadataRetriever.METADATA_KEY_ARTIST; +import static android.media.MediaMetadataRetriever.METADATA_KEY_CD_TRACK_NUMBER; +import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_RANGE; +import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_STANDARD; +import static android.media.MediaMetadataRetriever.METADATA_KEY_COLOR_TRANSFER; +import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPILATION; +import static android.media.MediaMetadataRetriever.METADATA_KEY_COMPOSER; +import static android.media.MediaMetadataRetriever.METADATA_KEY_DATE; +import static android.media.MediaMetadataRetriever.METADATA_KEY_DURATION; +import static android.media.MediaMetadataRetriever.METADATA_KEY_GENRE; +import static android.media.MediaMetadataRetriever.METADATA_KEY_IS_DRM; +import static android.media.MediaMetadataRetriever.METADATA_KEY_TITLE; +import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT; +import static android.media.MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH; +import static android.media.MediaMetadataRetriever.METADATA_KEY_YEAR; +import static android.provider.MediaStore.UNKNOWN_STRING; + +import android.annotation.NonNull; +import android.annotation.Nullable; +import android.content.ContentProviderClient; +import android.content.ContentProviderOperation; +import android.content.ContentProviderResult; +import android.content.ContentUris; +import android.content.Context; +import android.content.OperationApplicationException; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.media.ExifInterface; +import android.media.MediaFile; +import android.media.MediaMetadataRetriever; +import android.net.Uri; +import android.os.Environment; +import android.os.Trace; +import android.provider.MediaStore; +import android.provider.MediaStore.Audio.AudioColumns; +import android.provider.MediaStore.Files.FileColumns; +import android.provider.MediaStore.Images.ImageColumns; +import android.provider.MediaStore.MediaColumns; +import android.provider.MediaStore.Video.VideoColumns; +import android.text.TextUtils; +import android.util.ArrayMap; +import android.util.Log; +import android.util.LongArray; + +import com.android.providers.media.MediaProvider; + +import libcore.net.MimeUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Locale; +import java.util.TimeZone; + +/** + * Modern implementation of media scanner. + * <p> + * This is a bug-compatible reimplementation of the legacy media scanner, but + * written purely in managed code for better testability and long-term + * maintainability. + * <p> + * Initial tests shows it performing roughly on-par with the legacy scanner. + */ +public class ModernMediaScanner implements MediaScanner { + private static final String TAG = "ModernMediaScanner"; + private static final boolean LOGD = Log.isLoggable(TAG, Log.DEBUG); + private static final boolean LOGV = Log.isLoggable(TAG, Log.VERBOSE); + + // TODO: add playlist parsing + // TODO: add DRM support + + private static final SimpleDateFormat sDateFormat; + + static { + sDateFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss"); + sDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + private static final int BATCH_SIZE = 32; + + private final Context mContext; + private final MediaProvider mProvider; + + public ModernMediaScanner(Context context) { + mContext = context; + + try (ContentProviderClient cpc = context.getContentResolver() + .acquireContentProviderClient(MediaStore.AUTHORITY)) { + mProvider = (MediaProvider) cpc.getLocalContentProvider(); + } + } + + @Override + public Context getContext() { + return mContext; + } + + @Override + public void scanDirectory(File file) { + try (Scan scan = new Scan(file)) { + scan.run(); + } + } + + @Override + public Uri scanFile(File file) { + try (Scan scan = new Scan(file)) { + scan.run(); + return scan.mFirstResult; + } + } + + /** + * Individual scan request for a specific file or directory. When run it + * will traverse all included media files under the requested location, + * reconciling them against {@link MediaStore}. + */ + private class Scan implements Runnable, FileVisitor<Path>, AutoCloseable { + private final File mRoot; + private final String mVolumeName; + + private final ArrayList<ContentProviderOperation> mPending = new ArrayList<>(); + private LongArray mScannedIds = new LongArray(); + + private Uri mFirstResult; + + public Scan(File root) { + mRoot = root; + mVolumeName = MediaStore.getVolumeName(root); + } + + @Override + public void run() { + // First, scan everything that should be visible under requested + // location, tracking scanned IDs along the way + if (!isDirectoryHiddenRecursive(mRoot.isDirectory() ? mRoot : mRoot.getParentFile())) { + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "walkFileTree"); + try { + Files.walkFileTree(mRoot.toPath(), this); + } catch (IOException e) { + // This should never happen, so yell loudly + throw new IllegalStateException(e); + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + applyPending(); + } + + final long[] scannedIds = mScannedIds.toArray(); + Arrays.sort(scannedIds); + + // Second, clean up any deleted or hidden files, which are all items + // under requested location that weren't scanned above + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "clean"); + try (Cursor c = mProvider.query(MediaStore.Files.getContentUri(mVolumeName), + new String[] { FileColumns._ID }, + FileColumns.DATA + " LIKE ?", new String[] { mRoot.getAbsolutePath() + '%' }, + FileColumns._ID + " DESC")) { + while (c.moveToNext()) { + final long id = c.getLong(0); + if (Arrays.binarySearch(scannedIds, id) < 0) { + if (LOGV) Log.v(TAG, "Cleaning " + id); + final Uri uri = MediaStore.Files.getContentUri(mVolumeName, id).buildUpon() + .appendQueryParameter(MediaStore.PARAM_DELETE_DATA, "false") + .build(); + mPending.add(ContentProviderOperation.newDelete(uri).build()); + maybeApplyPending(); + } + } + applyPending(); + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + } + + @Override + public void close() { + // Sanity check that we drained any pending operations + if (!mPending.isEmpty()) { + throw new IllegalStateException(); + } + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (isDirectoryHidden(dir.toFile())) { + return FileVisitResult.SKIP_SUBTREE; + } + + // Scan this directory as a normal file so that "parent" database + // entries are created + return visitFile(dir, attrs); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (LOGV) Log.v(TAG, "Visiting " + file); + + // Skip files that have already been scanned, and which haven't + // changed since they were last scanned + final File realFile = file.toFile(); + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "checkChanged"); + try (Cursor c = mProvider.query(MediaStore.Files.getContentUri(mVolumeName), + new String[] { FileColumns._ID, FileColumns.DATE_MODIFIED, FileColumns.SIZE }, + FileColumns.DATA + "=?", new String[] { realFile.getAbsolutePath() }, null)) { + if (c.moveToFirst()) { + final long id = c.getLong(0); + final long dateModified = c.getLong(1); + final long size = c.getLong(2); + + final boolean sameTime = (attrs.lastModifiedTime().toMillis() + / 1000 == dateModified); + final boolean sameSize = (attrs.size() == size); + if (attrs.isDirectory() || (sameTime && sameSize)) { + if (LOGV) Log.v(TAG, "Skipping unchanged " + file); + mScannedIds.add(id); + return FileVisitResult.CONTINUE; + } + } + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + + final ContentProviderOperation op; + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "scanItem"); + try { + op = scanItem(file.toFile(), attrs, mVolumeName); + } finally { + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + if (op != null) { + mPending.add(op); + maybeApplyPending(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) + throws IOException { + Log.w(TAG, "Failed to visit " + file + ": " + exc); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) + throws IOException { + return FileVisitResult.CONTINUE; + } + + private void maybeApplyPending() { + if (mPending.size() > BATCH_SIZE) { + applyPending(); + } + } + + private void applyPending() { + Trace.traceBegin(Trace.TRACE_TAG_DATABASE, "applyPending"); + try { + for (ContentProviderResult res : mProvider.applyBatch(mPending)) { + if (res.uri != null) { + if (mFirstResult == null) { + mFirstResult = res.uri; + } + mScannedIds.add(ContentUris.parseId(res.uri)); + } + } + } catch (OperationApplicationException e) { + Log.w(TAG, "Failed to apply: " + e); + } finally { + mPending.clear(); + Trace.traceEnd(Trace.TRACE_TAG_DATABASE); + } + } + } + + /** + * Scan the requested file, returning a {@link ContentProviderOperation} + * containing all indexed metadata, suitable for passing to a + * {@link SQLiteDatabase#replace} operation. + */ + private static @Nullable ContentProviderOperation scanItem(File file, + BasicFileAttributes attrs, String volumeName) { + final String name = file.getName(); + if (name.startsWith(".")) { + if (LOGD) Log.d(TAG, "Ignoring hidden file: " + file); + return null; + } + + try { + final String mimeType; + if (attrs.isDirectory()) { + mimeType = null; + } else { + mimeType = MimeUtils.guessMimeTypeFromExtension(extractExtension(file)); + } + + if (attrs.isDirectory()) { + return scanItemDirectory(file, attrs, mimeType, volumeName); + } else if (MediaFile.isAudioMimeType(mimeType)) { + return scanItemAudio(file, attrs, mimeType, volumeName); + } else if (MediaFile.isPlayListMimeType(mimeType)) { + return scanItemPlaylist(file, attrs, mimeType, volumeName); + } else if (MediaFile.isVideoMimeType(mimeType)) { + return scanItemVideo(file, attrs, mimeType, volumeName); + } else if (MediaFile.isImageMimeType(mimeType)) { + return scanItemImage(file, attrs, mimeType, volumeName); + } else { + if (LOGD) Log.d(TAG, "Ignoring unsupported file: " + file); + return null; + } + } catch (IOException e) { + if (LOGD) Log.d(TAG, "Ignoring troubled file: " + file, e); + return null; + } + } + + /** + * Populate the given {@link ContentProviderOperation} with the generic + * {@link MediaColumns} values that can be determined directly from the file + * or its attributes. + */ + private static void scanItemGeneric(ContentProviderOperation.Builder op, File file, + BasicFileAttributes attrs, String mimeType) { + op.withValue(MediaColumns.DATA, file.getAbsolutePath()); + op.withValue(MediaColumns.SIZE, attrs.size()); + op.withValue(MediaColumns.TITLE, extractName(file)); + op.withValue(MediaColumns.DATE_MODIFIED, attrs.lastModifiedTime().toMillis() / 1000); + op.withValue(MediaColumns.MIME_TYPE, mimeType); + op.withValue(MediaColumns.IS_DRM, 0); + op.withValue(MediaColumns.WIDTH, null); + op.withValue(MediaColumns.HEIGHT, null); + } + + private static @NonNull ContentProviderOperation scanItemDirectory(File file, + BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { + final ContentProviderOperation.Builder op = ContentProviderOperation + .newInsert(MediaStore.Files.getContentUri(volumeName)); + try { + scanItemGeneric(op, file, attrs, mimeType); + op.withValue(FileColumns.MEDIA_TYPE, 0); + } catch (Exception e) { + throw new IOException(e); + } + return op.build(); + } + + private static ArrayMap<String, String> sAudioTypes = new ArrayMap<>(); + + static { + sAudioTypes.put(Environment.DIRECTORY_RINGTONES, AudioColumns.IS_RINGTONE); + sAudioTypes.put(Environment.DIRECTORY_NOTIFICATIONS, AudioColumns.IS_NOTIFICATION); + sAudioTypes.put(Environment.DIRECTORY_ALARMS, AudioColumns.IS_ALARM); + sAudioTypes.put(Environment.DIRECTORY_PODCASTS, AudioColumns.IS_PODCAST); + sAudioTypes.put(Environment.DIRECTORY_AUDIOBOOKS, AudioColumns.IS_AUDIOBOOK); + sAudioTypes.put(Environment.DIRECTORY_MUSIC, AudioColumns.IS_MUSIC); + } + + private static @NonNull ContentProviderOperation scanItemAudio(File file, + BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { + final ContentProviderOperation.Builder op = ContentProviderOperation + .newInsert(MediaStore.Audio.Media.getContentUri(volumeName)); + try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { + mmr.setDataSource(file.getAbsolutePath()); + + scanItemGeneric(op, file, attrs, mimeType); + + op.withValue(MediaColumns.TITLE, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_TITLE), extractName(file))); + op.withValue(MediaColumns.IS_DRM, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_IS_DRM), 0)); + + op.withValue(AudioColumns.DURATION, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_DURATION), null)); + op.withValue(AudioColumns.ARTIST, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_ARTIST), UNKNOWN_STRING)); + op.withValue(AudioColumns.ALBUM_ARTIST, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_ALBUMARTIST), null)); + op.withValue(AudioColumns.COMPILATION, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_COMPILATION), null)); + op.withValue(AudioColumns.COMPOSER, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_COMPOSER), null)); + op.withValue(AudioColumns.ALBUM, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_ALBUM), UNKNOWN_STRING)); + op.withValue(AudioColumns.TRACK, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_CD_TRACK_NUMBER), null)); + op.withValue(AudioColumns.YEAR, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_YEAR), null)); + + final String lowPath = file.getAbsolutePath().toLowerCase(Locale.ROOT); + boolean anyMatch = false; + for (int i = 0; i < sAudioTypes.size(); i++) { + final boolean match = lowPath + .contains('/' + sAudioTypes.keyAt(i).toLowerCase(Locale.ROOT) + '/'); + op.withValue(sAudioTypes.valueAt(i), match ? 1 : 0); + anyMatch |= match; + } + if (!anyMatch) { + op.withValue(AudioColumns.IS_MUSIC, 1); + } + + op.withValue(AudioColumns.GENRE, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_GENRE), null)); + } catch (Exception e) { + throw new IOException(e); + } + return op.build(); + } + + private static @NonNull ContentProviderOperation scanItemPlaylist(File file, + BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { + final ContentProviderOperation.Builder op = ContentProviderOperation + .newInsert(MediaStore.Audio.Playlists.getContentUri(volumeName)); + try { + scanItemGeneric(op, file, attrs, mimeType); + } catch (Exception e) { + throw new IOException(e); + } + return op.build(); + } + + private static @NonNull ContentProviderOperation scanItemVideo(File file, + BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { + final ContentProviderOperation.Builder op = ContentProviderOperation + .newInsert(MediaStore.Video.Media.getContentUri(volumeName)); + try (MediaMetadataRetriever mmr = new MediaMetadataRetriever()) { + mmr.setDataSource(file.getAbsolutePath()); + + scanItemGeneric(op, file, attrs, mimeType); + + op.withValue(MediaColumns.TITLE, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_TITLE), extractName(file))); + op.withValue(MediaColumns.IS_DRM, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_IS_DRM), 0)); + op.withValue(MediaColumns.WIDTH, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH), null)); + op.withValue(MediaColumns.HEIGHT, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT), null)); + + op.withValue(VideoColumns.DURATION, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_DURATION), null)); + op.withValue(VideoColumns.ARTIST, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_ARTIST), UNKNOWN_STRING)); + op.withValue(VideoColumns.ALBUM, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_ALBUM), + file.getParentFile().getName())); + op.withValue(VideoColumns.RESOLUTION, mmr.extractMetadata(METADATA_KEY_VIDEO_WIDTH) + + "x" + mmr.extractMetadata(METADATA_KEY_VIDEO_HEIGHT)); + op.withValue(VideoColumns.DESCRIPTION, null); + op.withValue(VideoColumns.DATE_TAKEN, + parseDate(mmr.extractMetadata(METADATA_KEY_DATE), + attrs.creationTime().toMillis())); + op.withValue(VideoColumns.COLOR_STANDARD, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_COLOR_STANDARD), null)); + op.withValue(VideoColumns.COLOR_TRANSFER, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_COLOR_TRANSFER), null)); + op.withValue(VideoColumns.COLOR_RANGE, + defeatEmpty(mmr.extractMetadata(METADATA_KEY_COLOR_RANGE), null)); + } catch (Exception e) { + throw new IOException(e); + } + return op.build(); + } + + private static @NonNull ContentProviderOperation scanItemImage(File file, + BasicFileAttributes attrs, String mimeType, String volumeName) throws IOException { + final ContentProviderOperation.Builder op = ContentProviderOperation + .newInsert(MediaStore.Images.Media.getContentUri(volumeName)); + try { + final ExifInterface exif = new ExifInterface(file); + + scanItemGeneric(op, file, attrs, mimeType); + + op.withValue(MediaColumns.WIDTH, + defeatEmpty(exif.getAttribute(ExifInterface.TAG_IMAGE_WIDTH), null)); + op.withValue(MediaColumns.HEIGHT, + defeatEmpty(exif.getAttribute(ExifInterface.TAG_IMAGE_LENGTH), null)); + + op.withValue(ImageColumns.DESCRIPTION, + defeatEmpty(exif.getAttribute(ExifInterface.TAG_IMAGE_DESCRIPTION), null)); + op.withValue(ImageColumns.DATE_TAKEN, + defeatEmpty(exif.getGpsDateTime(), exif.getDateTime())); + op.withValue(ImageColumns.ORIENTATION, + parseOrientation(exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, -1))); + } catch (Exception e) { + throw new IOException(e); + } + return op.build(); + } + + static String extractExtension(File file) { + final String name = file.getName(); + final int lastDot = name.lastIndexOf('.'); + return (lastDot == -1) ? null : name.substring(lastDot + 1); + } + + static String extractName(File file) { + final String name = file.getName(); + final int lastDot = name.lastIndexOf('.'); + return (lastDot == -1) ? name : name.substring(0, lastDot); + } + + private static Object defeatEmpty(String value, Object defaultValue) { + return TextUtils.isEmpty(value) ? defaultValue : value; + } + + private static long defeatEmpty(long value, long defaultValue) { + return (value == -1) ? defaultValue : value; + } + + private static int parseOrientation(int orientation) { + switch (orientation) { + case ExifInterface.ORIENTATION_ROTATE_90: return 90; + case ExifInterface.ORIENTATION_ROTATE_180: return 180; + case ExifInterface.ORIENTATION_ROTATE_270: return 270; + default: return 0; + } + } + + private static long parseDate(String date, long defaultValue) { + try { + final long value = sDateFormat.parse(date).getTime(); + return (value > 0) ? value : defaultValue; + } catch (ParseException e) { + return defaultValue; + } + } + + /** + * Test if any parents of given directory should be considered hidden. + */ + static boolean isDirectoryHiddenRecursive(File dir) { + while (dir != null) { + if (isDirectoryHidden(dir)) { + return true; + } + dir = dir.getParentFile(); + } + return false; + } + + /** + * Test if this given directory should be considered hidden. + */ + static boolean isDirectoryHidden(File dir) { + final String name = dir.getName(); + if (name.startsWith(".")) { + return true; + } + if (new File(dir, ".nomedia").exists()) { + return true; + } + return false; + } +} diff --git a/tests/res/raw/test_audio.mp3 b/tests/res/raw/test_audio.mp3 Binary files differnew file mode 100644 index 000000000..4fe922833 --- /dev/null +++ b/tests/res/raw/test_audio.mp3 diff --git a/tests/res/raw/test_image.jpg b/tests/res/raw/test_image.jpg Binary files differnew file mode 100644 index 000000000..cfe300f4e --- /dev/null +++ b/tests/res/raw/test_image.jpg diff --git a/tests/res/raw/test_video.mp4 b/tests/res/raw/test_video.mp4 Binary files differnew file mode 100644 index 000000000..ab95ac07d --- /dev/null +++ b/tests/res/raw/test_video.mp4 diff --git a/tests/src/com/android/providers/media/MediaScannerTest.java b/tests/src/com/android/providers/media/MediaScannerTest.java new file mode 100644 index 000000000..7ecffa765 --- /dev/null +++ b/tests/src/com/android/providers/media/MediaScannerTest.java @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.providers.media; + +import static org.junit.Assert.assertEquals; + +import android.content.ContentResolver; +import android.content.Context; +import android.content.ContextWrapper; +import android.content.pm.ProviderInfo; +import android.database.Cursor; +import android.database.DatabaseUtils; +import android.net.Uri; +import android.os.Bundle; +import android.os.Environment; +import android.os.FileUtils; +import android.os.SystemClock; +import android.provider.BaseColumns; +import android.provider.MediaStore; +import android.provider.MediaStore.MediaColumns; +import android.provider.Settings; +import android.test.mock.MockContentProvider; +import android.test.mock.MockContentResolver; +import android.util.Log; + +import com.android.providers.media.scan.LegacyMediaScanner; +import com.android.providers.media.scan.MediaScanner; +import com.android.providers.media.scan.ModernMediaScanner; +import com.android.providers.media.tests.R; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; + +import androidx.test.InstrumentationRegistry; +import androidx.test.runner.AndroidJUnit4; + +@RunWith(AndroidJUnit4.class) +public class MediaScannerTest { + private static final String TAG = "MediaScannerTest"; + + // TODO: scan directory-vs-files and confirm identical results + + private class IsolatedContext extends ContextWrapper { + private final File mDir; + private final MockContentResolver mResolver; + private final MediaProvider mProvider; + + public IsolatedContext(Context base, String tag) { + super(base); + mDir = new File(base.getFilesDir(), tag); + mDir.mkdirs(); + FileUtils.deleteContents(mDir); + + mResolver = new MockContentResolver(this); + + final ProviderInfo info = base.getPackageManager() + .resolveContentProvider(MediaStore.AUTHORITY, 0); + mProvider = new MediaProvider(); + mProvider.attachInfo(this, info); + + mResolver.addProvider(MediaStore.AUTHORITY, mProvider); + mResolver.addProvider(Settings.AUTHORITY, new MockContentProvider() { + @Override + public Bundle call(String method, String request, Bundle args) { + return Bundle.EMPTY; + } + }); + } + + @Override + public File getDatabasePath(String name) { + return new File(mDir, name); + } + + @Override + public ContentResolver getContentResolver() { + return mResolver; + } + } + + private MediaScanner mLegacy; + private MediaScanner mModern; + + @Before + public void setUp() { + final Context context = InstrumentationRegistry.getTargetContext(); + + mLegacy = new LegacyMediaScanner(new IsolatedContext(context, "legacy")); + mModern = new ModernMediaScanner(new IsolatedContext(context, "modern")); + } + + /** + * Ask both legacy and modern scanners to example sample files and assert + * the resulting database modifications are identical. + */ + @Test + public void testCorrectness() throws Exception { + final File dir = Environment.getExternalStorageDirectory(); + stage(R.raw.test_audio, new File(dir, "test.mp3")); + stage(R.raw.test_video, new File(dir, "test.mp4")); + stage(R.raw.test_image, new File(dir, "test.jpg")); + + // Execute both scanners in isolation + scanDirectory(mLegacy, dir, "legacy"); + scanDirectory(mModern, dir, "modern"); + + // Confirm that they both agree on scanned details + for (Uri uri : new Uri[] { + MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, + MediaStore.Video.Media.EXTERNAL_CONTENT_URI, + MediaStore.Images.Media.EXTERNAL_CONTENT_URI, + }) { + final Context legacyContext = mLegacy.getContext(); + final Context modernContext = mModern.getContext(); + try (Cursor cl = legacyContext.getContentResolver().query(uri, null, null, null); + Cursor cm = modernContext.getContentResolver().query(uri, null, null, null)) { + try { + // Must have same count + assertEquals(cl.getCount(), cm.getCount()); + + while (cl.moveToNext() && cm.moveToNext()) { + for (int i = 0; i < cl.getColumnCount(); i++) { + final String columnName = cl.getColumnName(i); + if (columnName.equals(MediaColumns._ID)) continue; + if (columnName.equals(MediaColumns.DATE_ADDED)) continue; + + // Must have same name + assertEquals(cl.getColumnName(i), cm.getColumnName(i)); + // Must have same data types + assertEquals(columnName + " type", + cl.getType(i), cm.getType(i)); + // Must have same contents + assertEquals(columnName + " value", + cl.getString(i), cm.getString(i)); + } + } + } catch (AssertionError e) { + Log.d(TAG, "Legacy:"); + DatabaseUtils.dumpCursor(cl); + Log.d(TAG, "Modern:"); + DatabaseUtils.dumpCursor(cm); + throw e; + } + } + } + } + + @Test + public void testSpeed_Legacy() throws Exception { + testSpeed(mLegacy); + } + + @Test + public void testSpeed_Modern() throws Exception { + testSpeed(mModern); + } + + private void testSpeed(MediaScanner scanner) throws IOException { + final File scanDir = Environment.getExternalStorageDirectory(); + final File dir = new File(Environment.getExternalStorageDirectory(), + "test" + System.nanoTime()); + + stage(dir, 4, 3); + scanDirectory(scanner, scanDir, "Initial"); + scanDirectory(scanner, scanDir, "No-op"); + + FileUtils.deleteContentsAndDir(dir); + scanDirectory(scanner, scanDir, "Clean"); + } + + private static void scanDirectory(MediaScanner scanner, File dir, String tag) { + final Context context = scanner.getContext(); + final long beforeTime = SystemClock.elapsedRealtime(); + final int[] beforeCounts = getCounts(context); + + scanner.scanDirectory(dir); + + final long deltaTime = SystemClock.elapsedRealtime() - beforeTime; + final int[] deltaCounts = subtract(getCounts(context), beforeCounts); + Log.i(TAG, "Scan " + tag + ": " + deltaTime + "ms " + Arrays.toString(deltaCounts)); + } + + private static int[] subtract(int[] a, int[] b) { + final int[] c = new int[a.length]; + for (int i = 0; i < a.length; i++) { + c[i] = a[i] - b[i]; + } + return c; + } + + private static int[] getCounts(Context context) { + return new int[] { + getCount(context, MediaStore.Files.EXTERNAL_CONTENT_URI), + getCount(context, MediaStore.Audio.Media.EXTERNAL_CONTENT_URI), + getCount(context, MediaStore.Video.Media.EXTERNAL_CONTENT_URI), + getCount(context, MediaStore.Images.Media.EXTERNAL_CONTENT_URI), + }; + } + + private static int getCount(Context context, Uri uri) { + try (Cursor c = context.getContentResolver().query(uri, + new String[] { BaseColumns._ID }, null, null)) { + return c.getCount(); + } + } + + private static void stage(File dir, int deep, int wide) throws IOException { + dir.mkdirs(); + + if (deep > 0) { + stage(new File(dir, "dir" + System.nanoTime()), deep - 1, wide * 2); + } + + for (int i = 0; i < wide; i++) { + stage(R.raw.test_image, new File(dir, System.nanoTime() + ".jpg")); + stage(R.raw.test_video, new File(dir, System.nanoTime() + ".mp4")); + } + } + + private static void stage(int resId, File file) throws IOException { + final Context context = InstrumentationRegistry.getContext(); + try (InputStream source = context.getResources().openRawResource(resId); + OutputStream target = new FileOutputStream(file)) { + FileUtils.copy(source, target); + } + } +} |
