--- toolkit/components/url-classifier/Makefile.in | 3 toolkit/components/url-classifier/pref/url-classifier.js | 8 + toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp | 62 +++++++--- 3 files changed, 59 insertions(+), 14 deletions(-) Index: mozilla/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp =================================================================== --- mozilla.orig/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp +++ mozilla/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp @@ -122,17 +122,17 @@ // accessing the same database (such as between stable versions of the // platform). #define DATABASE_FILENAME "urlclassifier3.sqlite" // The implementation version is updated during development when we // want to change schema, or to recover from updating bugs. When an // implementation version change is detected, the database is scrapped // and we start over. -#define IMPLEMENTATION_VERSION 3 +#define IMPLEMENTATION_VERSION 4 #define MAX_HOST_COMPONENTS 5 #define MAX_PATH_COMPONENTS 4 // Updates will fail if fed chunks larger than this #define MAX_CHUNK_SIZE (1024 * 1024) // Prefs for implementing nsIURIClassifier to block page loads @@ -140,28 +140,35 @@ #define CHECK_MALWARE_DEFAULT PR_FALSE #define CHECK_PHISHING_PREF "browser.safebrowsing.enabled" #define CHECK_PHISHING_DEFAULT PR_FALSE #define GETHASH_NOISE_PREF "urlclassifier.gethashnoise" #define GETHASH_NOISE_DEFAULT 4 +#define UPDATE_CACHE_SIZE_PREF "urlclassifier.updatecachemax" +#define UPDATE_CACHE_SIZE_DEFAULT -1 + +#define PAGE_SIZE 4096 + class nsUrlClassifierDBServiceWorker; // Singleton instance. static nsUrlClassifierDBService* sUrlClassifierDBService; // Thread that we do the updates on. static nsIThread* gDbBackgroundThread = nsnull; // Once we've committed to shutting down, don't do work in the background // thread. static PRBool gShuttingDownThread = PR_FALSE; +static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT; + // ------------------------------------------------------------------------- // Hash class implementation // A convenience wrapper around the potentially-truncated hash for a // domain or fragment. template struct nsUrlClassifierHash @@ -1121,16 +1128,17 @@ // We receive data in small chunks that may be broken in the middle of // a line. So we save the last partial line here. nsCString mPendingStreamUpdate; PRInt32 mUpdateWait; PRBool mResetRequested; + PRBool mGrewCache; enum { STATE_LINE, STATE_CHUNK } mState; enum { CHUNK_ADD, @@ -1186,16 +1194,17 @@ }; NS_IMPL_THREADSAFE_ISUPPORTS1(nsUrlClassifierDBServiceWorker, nsIUrlClassifierDBServiceWorker) nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker() : mUpdateWait(0) , mResetRequested(PR_FALSE) + , mGrewCache(PR_FALSE) , mState(STATE_LINE) , mChunkType(CHUNK_ADD) , mChunkNum(0) , mHashSize(0) , mChunkLen(0) , mUpdateTableId(0) , mUpdateStatus(NS_OK) , mInStream(PR_FALSE) @@ -2120,23 +2129,23 @@ if (i != 0) { chunkStr.Append(','); } chunkStr.AppendInt(chunks[i]); PRUint32 first = i; PRUint32 last = first; i++; - while (i < chunks.Length() && chunks[i] == chunks[i - 1] + 1) { - last = chunks[i++]; + while (i < chunks.Length() && (chunks[i] == chunks[i - 1] + 1 || chunks[i] == chunks[i - 1])) { + last = i++; } if (last != first) { chunkStr.Append('-'); - chunkStr.AppendInt(last); + chunkStr.AppendInt(chunks[last]); } } return NS_OK; } nsresult @@ -2270,17 +2279,17 @@ { #if defined(PR_LOGGING) PRIntervalTime clockStart = 0; if (LOG_ENABLED()) { clockStart = PR_IntervalNow(); } #endif - LOG(("Adding %d entries to chunk %d", entries.Length(), chunkNum)); + LOG(("Adding %d entries to chunk %d in table %d", entries.Length(), chunkNum, tableId)); nsresult rv = CacheChunkLists(tableId, PR_TRUE, PR_FALSE); NS_ENSURE_SUCCESS(rv, rv); mCachedAddChunks.AppendElement(chunkNum); nsTArray entryIDs; nsAutoTArray subEntries; @@ -2650,16 +2659,28 @@ } if (transaction) { NS_WARNING("Transaction already in progress in nsUrlClassifierDBServiceWorker::BeginUpdate. Cancelling update."); mUpdateStatus = NS_ERROR_FAILURE; return rv; } + if (gUpdateCacheSize > 0) { + PRUint32 cachePages = gUpdateCacheSize / PAGE_SIZE; + nsCAutoString cacheSizePragma("PRAGMA cache_size="); + cacheSizePragma.AppendInt(cachePages); + rv = mConnection->ExecuteSimpleSQL(cacheSizePragma); + if (NS_FAILED(rv)) { + mUpdateStatus = rv; + return rv; + } + mGrewCache = PR_TRUE; + } + rv = mConnection->BeginTransaction(); if (NS_FAILED(rv)) { mUpdateStatus = rv; return rv; } mUpdateObserver = observer; @@ -2875,16 +2896,23 @@ ResetUpdate(); // It's important that we only reset the database if the update was // successful, otherwise unauthenticated updates could cause a // database reset. if (NS_SUCCEEDED(mUpdateStatus) && resetRequested) { ResetDatabase(); + } else if (mGrewCache) { + // During the update we increased the page cache to bigger than we + // want to keep around. At the moment, the only reliable way to make + // sure that the page cache is freed is to reopen the connection. + mGrewCache = PR_FALSE; + CloseDb(); + OpenDb(); } return NS_OK; } NS_IMETHODIMP nsUrlClassifierDBServiceWorker::ResetDatabase() { @@ -3020,29 +3048,28 @@ newDB = PR_TRUE; rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection)); NS_ENSURE_SUCCESS(rv, rv); } } - if (newDB) { - rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION); - NS_ENSURE_SUCCESS(rv, rv); - } - - rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF")); + nsCAutoString cacheSizePragma("PRAGMA page_size="); + cacheSizePragma.AppendInt(PAGE_SIZE); + rv = connection->ExecuteSimpleSQL(cacheSizePragma); NS_ENSURE_SUCCESS(rv, rv); - rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA page_size=4096")); + rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF")); NS_ENSURE_SUCCESS(rv, rv); - rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA default_page_size=4096")); - NS_ENSURE_SUCCESS(rv, rv); + if (newDB) { + rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION); + NS_ENSURE_SUCCESS(rv, rv); + } // Create the table rv = MaybeCreateTables(connection); NS_ENSURE_SUCCESS(rv, rv); rv = mMainStore.Init(this, connection, NS_LITERAL_CSTRING("moz_classifier")); NS_ENSURE_SUCCESS(rv, rv); @@ -3490,16 +3517,19 @@ rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool); mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT; prefs->AddObserver(CHECK_PHISHING_PREF, this, PR_FALSE); if (NS_FAILED(prefs->GetIntPref(GETHASH_NOISE_PREF, &gethashNoise))) { gethashNoise = GETHASH_NOISE_DEFAULT; } + PRInt32 tmpint; + rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint); + PR_AtomicSet(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT); } // Start the background thread. rv = NS_NewThread(&gDbBackgroundThread); if (NS_FAILED(rv)) return rv; mWorker = new nsUrlClassifierDBServiceWorker(); @@ -3763,16 +3793,20 @@ if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) { PRBool tmpbool; rv = prefs->GetBoolPref(CHECK_MALWARE_PREF, &tmpbool); mCheckMalware = NS_SUCCEEDED(rv) ? tmpbool : CHECK_MALWARE_DEFAULT; } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) { PRBool tmpbool; rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool); mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT; + } else if (NS_LITERAL_STRING(UPDATE_CACHE_SIZE_PREF).Equals(aData)) { + PRInt32 tmpint; + rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint); + PR_AtomicSet(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT); } } else if (!strcmp(aTopic, "profile-before-change") || !strcmp(aTopic, "xpcom-shutdown-threads")) { Shutdown(); } else { return NS_ERROR_UNEXPECTED; } Index: mozilla/toolkit/components/url-classifier/pref/url-classifier.js =================================================================== --- /dev/null +++ mozilla/toolkit/components/url-classifier/pref/url-classifier.js @@ -0,0 +1,8 @@ + +// Maximum size of the sqlite3 cache during an update, in bytes +#ifdef MOZ_WIDGET_GTK2 +pref("urlclassifier.updatecachemax", 104857600); +#else +pref("urlclassifier.updatecachemax", -1); +#endif + Index: mozilla/toolkit/components/url-classifier/Makefile.in =================================================================== --- mozilla.orig/toolkit/components/url-classifier/Makefile.in +++ mozilla/toolkit/components/url-classifier/Makefile.in @@ -46,10 +46,13 @@ DIRS = public src ifdef ENABLE_TESTS ifndef MOZ_ENABLE_LIBXUL TOOL_DIRS += tests endif endif +PREF_JS_EXPORTS = $(srcdir)/pref/url-classifier.js +GARBAGE += $(addprefix $(DIST)/bin/defaults/pref/,url-classifier.js) + include $(topsrcdir)/config/rules.mk