From 4b59d8badf35f84e1525620e837b378f0899aac9 Mon Sep 17 00:00:00 2001 From: Jeff Kaufman Date: Wed, 28 Nov 2012 15:44:05 -0500 Subject: [PATCH] caching: support memcached Merge in Otto's #65 porting the apache memcached implementation to nginx. --- README.md | 15 +++ config | 12 +- src/ngx_cache.cc | 102 ++++++++++++++ src/ngx_cache.h | 75 +++++++++++ src/ngx_pagespeed.cc | 18 +++ src/ngx_rewrite_driver_factory.cc | 215 +++++++++++++++++++++++++----- src/ngx_rewrite_driver_factory.h | 66 ++++++++- src/ngx_rewrite_options.cc | 6 + src/ngx_rewrite_options.h | 24 +++- 9 files changed, 496 insertions(+), 37 deletions(-) create mode 100644 src/ngx_cache.cc create mode 100644 src/ngx_cache.h diff --git a/README.md b/README.md index d3c2e6bc9..5cfd1f90e 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,21 @@ and then eventually: along with a failing test because ngx_pagespeed is not yet complete. +#### Testing with memcached + +Start an memcached server: + + memcached -p 11213 + +To the configuration above add to the main or server block: + + pagespeed MemcachedServers "localhost:11213"; + pagespeed MemcachedThreads 1; + +Then run the system test: + + /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050 + ## Configuration Once configuration is complete, any mod_pagespeed configuration directive should diff --git a/config b/config index 592a934f3..600dc9783 100644 --- a/config +++ b/config @@ -89,17 +89,23 @@ ngx_feature_test=" if [ $ngx_found = yes ]; then ps_src="$ngx_addon_dir/src" ngx_addon_name=ngx_pagespeed + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/apr_thread_compatible_pool.cc" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/serf_url_async_fetcher.cc" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/apr_mem_cache.cc" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/util/key_value_codec.cc" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/third_party/aprutil/apr_memcache2.c" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_pagespeed.cc" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_rewrite_driver_factory.cc" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_rewrite_options.cc" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_server_context.cc" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_base_fetch.cc" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_cache.cc" HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name" CORE_LIBS="$CORE_LIBS $pagespeed_libs - $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/serf/libserf.a - $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/apr/libapr.a - $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/aprutil/libaprutil.a" + $mod_pagespeed_dir/out/Release/obj.target/third_party/serf/libserf.a + $mod_pagespeed_dir/out/Release/obj.target/third_party/aprutil/libaprutil.a + $mod_pagespeed_dir/out/Release/obj.target/third_party/apr/libapr.a" CORE_INCS="$CORE_INCS $pagespeed_include" else cat << END diff --git a/src/ngx_cache.cc b/src/ngx_cache.cc new file mode 100644 index 000000000..5b7a6e5ea --- /dev/null +++ b/src/ngx_cache.cc @@ -0,0 +1,102 @@ +// Copyright 2012 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: oschaaf@gmail.com (Otto van der Schaaf) + +#include "ngx_cache.h" +#include "ngx_rewrite_options.h" +#include "ngx_rewrite_driver_factory.h" +#include "net/instaweb/util/public/cache_interface.h" +#include "net/instaweb/util/public/cache_stats.h" +#include "net/instaweb/util/public/file_cache.h" +#include "net/instaweb/util/public/file_system_lock_manager.h" +#include "net/instaweb/util/public/lru_cache.h" +#include "net/instaweb/util/public/message_handler.h" +#include "net/instaweb/util/public/shared_mem_lock_manager.h" +#include "net/instaweb/util/public/thread_system.h" +#include "net/instaweb/util/public/threadsafe_cache.h" + +namespace net_instaweb { + +const char NgxCache::kFileCache[] = "file_cache"; +const char NgxCache::kLruCache[] = "lru_cache"; + +// TODO(oschaaf): refactor this to share as much as possible +// with apache_cache.cc +// The NgxCache shares a file cache per path, with an optional +// LRU Cache +NgxCache::NgxCache(const StringPiece& path, + const NgxRewriteOptions& config, + NgxRewriteDriverFactory* factory) + : path_(path.data(), path.size()), + factory_(factory), + lock_manager_(NULL), + file_cache_(NULL) { + if (config.use_shared_mem_locking()) { + shared_mem_lock_manager_.reset(new SharedMemLockManager( + factory->shared_mem_runtime(), StrCat(path, "/named_locks"), + factory->scheduler(), factory->hasher(), factory->message_handler())); + lock_manager_ = shared_mem_lock_manager_.get(); + } else { + FallBackToFileBasedLocking(); + } + + FileCache::CachePolicy* policy = new FileCache::CachePolicy( + factory->timer(), + factory->hasher(), + config.file_cache_clean_interval_ms(), + config.file_cache_clean_size_kb() * 1024, + config.file_cache_clean_inode_limit()); + file_cache_ = new FileCache( + config.file_cache_path(), factory->file_system(), NULL, + factory->filename_encoder(), policy, factory->message_handler()); + l2_cache_.reset(new CacheStats(kFileCache, file_cache_, factory->timer(), + factory->statistics())); + + if (config.lru_cache_kb_per_process() != 0) { + LRUCache* lru_cache = new LRUCache( + config.lru_cache_kb_per_process() * 1024); + + // We only add the threadsafe-wrapper to the LRUCache. The FileCache + // is naturally thread-safe because it's got no writable member variables. + // And surrounding that slower-running class with a mutex would likely + // cause contention. + ThreadsafeCache* ts_cache = + new ThreadsafeCache(lru_cache, factory->thread_system()->NewMutex()); + // TODO(oschaaf): Non-portable (though most major compilers accept it). +#if CACHE_STATISTICS + l1_cache_.reset(new CacheStats(kLruCache, ts_cache, factory->timer(), + factory->statistics())); +#else + l1_cache_.reset(ts_cache); +#endif + } +} + +NgxCache::~NgxCache() { +} + +// TODO(oschaaf): see rootinit/childinit from ApacheCache.cc + +void NgxCache::FallBackToFileBasedLocking() { + if ((shared_mem_lock_manager_.get() != NULL) || (lock_manager_ == NULL)) { + shared_mem_lock_manager_.reset(NULL); + file_system_lock_manager_.reset(new FileSystemLockManager( + factory_->file_system(), path_, + factory_->scheduler(), factory_->message_handler())); + lock_manager_ = file_system_lock_manager_.get(); + } +} + +} // namespace net_instaweb diff --git a/src/ngx_cache.h b/src/ngx_cache.h new file mode 100644 index 000000000..0378951f6 --- /dev/null +++ b/src/ngx_cache.h @@ -0,0 +1,75 @@ +// Copyright 2012 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: oschaaf@gmail.com (Otto van der Schaaf) + +#ifndef NGX_CACHE_H_ +#define NGX_CACHE_H_ + +#include "base/scoped_ptr.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + +namespace net_instaweb { + +class NgxRewriteOptions; +class NgxRewriteDriverFactory; +class CacheInterface; +class FileCache; +class FileSystemLockManager; +class MessageHandler; +class NamedLockManager; +class SharedMemLockManager; + +// The NgxCache encapsulates a cache-sharing model where a user specifies +// a file-cache path per virtual-host. With each file-cache object we keep +// a locking mechanism and an optional per-process LRUCache. +class NgxCache { + public: + static const char kFileCache[]; + static const char kLruCache[]; + + NgxCache(const StringPiece& path, + const NgxRewriteOptions& config, + NgxRewriteDriverFactory* factory); + ~NgxCache(); + CacheInterface* l1_cache() { return l1_cache_.get(); } + CacheInterface* l2_cache() { return l2_cache_.get(); } + NamedLockManager* lock_manager() { return lock_manager_; } + + void RootInit(); + void ChildInit(); + void GlobalCleanup(MessageHandler* handler); // only called in root process + + private: + void FallBackToFileBasedLocking(); + + GoogleString path_; + NgxRewriteDriverFactory* factory_; + scoped_ptr shared_mem_lock_manager_; + scoped_ptr file_system_lock_manager_; + NamedLockManager* lock_manager_; + FileCache* file_cache_; // owned by l2 cache + scoped_ptr l1_cache_; + scoped_ptr l2_cache_; +}; + +// CACHE_STATISTICS is #ifdef'd to facilitate experiments with whether +// tracking the detailed stats & histograms has a QPS impact. Set it +// to 0 to turn it off. +#define CACHE_STATISTICS 1 + +} // namespace net_instaweb + +#endif // NGX_CACHE_H_ diff --git a/src/ngx_pagespeed.cc b/src/ngx_pagespeed.cc index 185860d79..9155da213 100644 --- a/src/ngx_pagespeed.cc +++ b/src/ngx_pagespeed.cc @@ -280,6 +280,9 @@ ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf); +void +ps_ignore_sigpipe(); + ngx_command_t ps_commands[] = { { ngx_string("pagespeed"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1| @@ -300,6 +303,16 @@ ngx_command_t ps_commands[] = { ngx_null_command }; +void +ps_ignore_sigpipe() { + struct sigaction act; + ngx_memzero(&act, sizeof(act)); + act.sa_handler = SIG_IGN; + sigemptyset (&act.sa_mask); + act.sa_flags = 0; + sigaction (SIGPIPE, &act, NULL); +} + #define NGX_PAGESPEED_MAX_ARGS 10 char* ps_configure(ngx_conf_t* cf, @@ -430,6 +443,11 @@ ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) { // because if there are no server blocks with pagespeed configuration // directives then we don't want it initialized. if (cfg_m->driver_factory == NULL) { + // TODO(oschaaf): this ignores sigpipe messages from memcached. + // however, it would be better to not have those signals generated + // in the first place, as suppressing them this way may interfere + // with other modules that actually are interested in these signals + ps_ignore_sigpipe(); net_instaweb::NgxRewriteDriverFactory::Initialize(); // TODO(jefftk): We should call NgxRewriteDriverFactory::Terminate() when // we're done with it. That never happens, though, because this is the diff --git a/src/ngx_rewrite_driver_factory.cc b/src/ngx_rewrite_driver_factory.cc index 43884e897..1ebb1fac6 100644 --- a/src/ngx_rewrite_driver_factory.cc +++ b/src/ngx_rewrite_driver_factory.cc @@ -44,10 +44,19 @@ #include "net/instaweb/util/public/write_through_cache.h" #include "net/instaweb/http/public/http_cache.h" #include "net/instaweb/http/public/write_through_http_cache.h" - -// TODO(oschaaf): discuss the proper way to to this. -#include "net/instaweb/apache/apr_thread_compatible_pool.cc" -#include "net/instaweb/apache/serf_url_async_fetcher.cc" +#include "net/instaweb/apache/apr_thread_compatible_pool.h" +#include "net/instaweb/apache/serf_url_async_fetcher.h" +#include "net/instaweb/apache/apr_mem_cache.h" +#include "net/instaweb/util/public/null_shared_mem.h" +#include "net/instaweb/util/public/cache_copy.h" +#include "net/instaweb/util/public/async_cache.h" +#include "net/instaweb/util/public/cache_stats.h" +#include "net/instaweb/util/public/cache_batcher.h" +#include "net/instaweb/util/public/fallback_cache.h" +#include "ngx_cache.h" +#include "net/instaweb/apache/apr_thread_compatible_pool.h" +#include "net/instaweb/apache/serf_url_async_fetcher.h" +#include "net/instaweb/apache/apr_mem_cache.h" namespace net_instaweb { @@ -61,20 +70,42 @@ class UrlAsyncFetcher; class UrlFetcher; class Writer; -NgxRewriteDriverFactory::NgxRewriteDriverFactory() { +const char NgxRewriteDriverFactory::kMemcached[] = "memcached"; + +NgxRewriteDriverFactory::NgxRewriteDriverFactory() : + shared_mem_runtime_(new NullSharedMem()), + cache_hasher_(20) { RewriteDriverFactory::InitStats(&simple_stats_); SerfUrlAsyncFetcher::InitStats(&simple_stats_); + AprMemCache::InitStats(&simple_stats_); + CacheStats::InitStats(NgxCache::kFileCache, &simple_stats_); + CacheStats::InitStats(NgxCache::kLruCache, &simple_stats_); + CacheStats::InitStats(kMemcached, &simple_stats_); SetStatistics(&simple_stats_); timer_ = DefaultTimer(); apr_initialize(); apr_pool_create(&pool_,NULL); - InitializeDefaultOptions(); } NgxRewriteDriverFactory::~NgxRewriteDriverFactory() { delete timer_; + timer_ = NULL; slow_worker_->ShutDown(); + apr_pool_destroy(pool_); + pool_ = NULL; + + for (PathCacheMap::iterator p = path_cache_map_.begin(), + e = path_cache_map_.end(); p != e; ++p) { + NgxCache* cache = p->second; + defer_cleanup(new Deleter(cache)); + } + + for (MemcachedMap::iterator p = memcached_map_.begin(), + e = memcached_map_.end(); p != e; ++p) { + CacheInterface* memcached = p->second; + defer_cleanup(new Deleter(memcached)); + } } const char NgxRewriteDriverFactory::kStaticJavaScriptPrefix[] = @@ -128,35 +159,45 @@ void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) { NgxRewriteOptions* options = NgxRewriteOptions::DynamicCast( server_context->global_options()); - LRUCache* lru_cache = new LRUCache( - options->lru_cache_kb_per_process() * 1024); - CacheInterface* cache = new ThreadsafeCache( - lru_cache, thread_system()->NewMutex()); + NgxCache* ngx_cache = GetCache(options); + CacheInterface* l1_cache = ngx_cache->l1_cache(); + CacheInterface* l2_cache = ngx_cache->l2_cache(); + CacheInterface* memcached = GetMemcached(options, l2_cache); + if (memcached != NULL) { + // XXX(oschaaf): remove when done + l1_cache = NULL; + l2_cache = memcached; + server_context->set_owned_cache(memcached); + server_context->set_filesystem_metadata_cache( + new CacheCopy(GetFilesystemMetadataCache(options))); + } + Statistics* stats = server_context->statistics(); - FileCache::CachePolicy* policy = new FileCache::CachePolicy( - timer(), - hasher(), - options->file_cache_clean_interval_ms(), - options->file_cache_clean_size_kb() * 1024, - options->file_cache_clean_inode_limit()); + // TODO(jmarantz): consider moving ownership of the L1 cache into the + // factory, rather than having one per vhost. + // + // Note that a user can disable the L1 cache by setting its byte-count + // to 0, in which case we don't build the write-through mechanisms. + if (l1_cache == NULL) { + HTTPCache* http_cache = new HTTPCache(l2_cache, timer(), hasher(), stats); + server_context->set_http_cache(http_cache); + server_context->set_metadata_cache(new CacheCopy(l2_cache)); + server_context->MakePropertyCaches(l2_cache); + } else { + WriteThroughHTTPCache* write_through_http_cache = new WriteThroughHTTPCache( + l1_cache, l2_cache, timer(), hasher(), stats); + write_through_http_cache->set_cache1_limit(options->lru_cache_byte_limit()); + server_context->set_http_cache(write_through_http_cache); - FileCache* file_cache = new FileCache(options->file_cache_path(), - file_system(), NULL, - filename_encoder(), policy, - message_handler()); + WriteThroughCache* write_through_cache = new WriteThroughCache( + l1_cache, l2_cache); + write_through_cache->set_cache1_limit(options->lru_cache_byte_limit()); + server_context->set_metadata_cache(write_through_cache); + server_context->MakePropertyCaches(l2_cache); + } - slow_worker_.reset(new SlowWorker(thread_system())); - - WriteThroughHTTPCache* write_through_http_cache = new WriteThroughHTTPCache( - cache, file_cache, timer(), hasher(), statistics()); - write_through_http_cache->set_cache1_limit(options->lru_cache_byte_limit()); - server_context->set_http_cache(write_through_http_cache); - - WriteThroughCache* write_through_cache = new WriteThroughCache( - cache, file_cache); - write_through_cache->set_cache1_limit(options->lru_cache_byte_limit()); - server_context->set_metadata_cache(write_through_cache); - server_context->MakePropertyCaches(file_cache); + // TODO(oschaaf): see the property cache setup in the apache rewrite + // driver factory server_context->set_enable_property_cache(true); } @@ -173,4 +214,114 @@ void NgxRewriteDriverFactory::InitStaticJavascriptManager( static_js_manager->set_library_url_prefix(kStaticJavaScriptPrefix); } +NgxCache* NgxRewriteDriverFactory::GetCache(NgxRewriteOptions* options) { + const GoogleString& path = options->file_cache_path(); + std::pair result = path_cache_map_.insert( + PathCacheMap::value_type(path, static_cast(NULL))); + PathCacheMap::iterator iter = result.first; + if (result.second) { + iter->second = new NgxCache(path, *options, this); + } + return iter->second; +} + +AprMemCache* NgxRewriteDriverFactory::NewAprMemCache( + const GoogleString& spec) { + // TODO(oschaaf): determine a sensible limit + int thread_limit = 2; + return new AprMemCache(spec, thread_limit, &cache_hasher_, statistics(), + timer(), message_handler()); +} + +CacheInterface* NgxRewriteDriverFactory::GetMemcached( + NgxRewriteOptions* options, CacheInterface* l2_cache) { + CacheInterface* memcached = NULL; + + // Find a memcache that matches the current spec, or create a new one + // if needed. Note that this means that two different VirtualHost's will + // share a memcached if their specs are the same but will create their own + // if the specs are different. + if (!options->memcached_servers().empty()) { + const GoogleString& server_spec = options->memcached_servers(); + std::pair result = memcached_map_.insert( + MemcachedMap::value_type(server_spec, memcached)); + if (result.second) { + fprintf(stderr,"setting up memcached server [%s]\n",server_spec.c_str()); + AprMemCache* mem_cache = NewAprMemCache(server_spec); + + memcache_servers_.push_back(mem_cache); + + int num_threads = options->memcached_threads(); + if (num_threads != 0) { + if (memcached_pool_.get() == NULL) { + // Note -- we will use the first value of ModPagespeedMemCacheThreads + // that we see in a VirtualHost, ignoring later ones. + memcached_pool_.reset(new QueuedWorkerPool(num_threads, + thread_system())); + } + AsyncCache* async_cache = new AsyncCache(mem_cache, + memcached_pool_.get()); + async_caches_.push_back(async_cache); + memcached = async_cache; + } else { + message_handler()->Message(kWarning, + "Running memcached synchronously, this may hurt performance"); + memcached = mem_cache; + } + + // Put the batcher above the stats so that the stats sees the MultiGets + // and can show us the histogram of how they are sized. +#if CACHE_STATISTICS + memcached = new CacheStats(kMemcached, memcached, timer(), statistics()); +#endif + CacheBatcher* batcher = new CacheBatcher( + memcached, thread_system()->NewMutex(), statistics()); + if (num_threads != 0) { + batcher->set_max_parallel_lookups(num_threads); + } + memcached = batcher; + result.first->second = memcached; + + // TODO(oschaaf): should not connect to memcached here + bool connected = mem_cache->Connect(); + fprintf(stderr, + "connected to memcached backend: %s\n", connected ? "true": "false"); + + } else { + memcached = result.first->second; + } + + // Note that a distinct FallbackCache gets created for every VirtualHost + // that employs memcached, even if the memcached and file-cache + // specifications are identical. This does no harm, because there + // is no data in the cache object itself; just configuration. Sharing + // FallbackCache* objects would require making a map using the + // memcache & file-cache specs as a key, so it's simpler to make a new + // small FallbackCache object for each VirtualHost. + memcached = new FallbackCache(memcached, l2_cache, + AprMemCache::kValueSizeThreshold, + message_handler()); + } + return memcached; +} + +CacheInterface* NgxRewriteDriverFactory::GetFilesystemMetadataCache( + NgxRewriteOptions* options) { + // Reuse the memcached server(s) for the filesystem metadata cache. We need + // to search for our config's entry in the vector of servers (not the more + // obvious map) because the map's entries are wrapped in an AsyncCache, and + // the filesystem metadata cache requires a blocking cache (like memcached). + // Note that if we have a server spec we *know* it's in the searched vector. + // We perform a linear scan assuming that the searched set will be small + DCHECK_EQ(options->memcached_servers().empty(), memcache_servers_.empty()); + const GoogleString& server_spec = options->memcached_servers(); + for (int i = 0, n = memcache_servers_.size(); i < n; ++i) { + if (server_spec == memcache_servers_[i]->server_spec()) { + return memcache_servers_[i]; + } + } + + return NULL; +} + } // namespace net_instaweb diff --git a/src/ngx_rewrite_driver_factory.h b/src/ngx_rewrite_driver_factory.h index 536aed201..6afeeb424 100644 --- a/src/ngx_rewrite_driver_factory.h +++ b/src/ngx_rewrite_driver_factory.h @@ -21,17 +21,31 @@ #include "base/scoped_ptr.h" #include "net/instaweb/rewriter/public/rewrite_driver_factory.h" +#include "net/instaweb/util/public/md5_hasher.h" #include "net/instaweb/util/public/simple_stats.h" #include "apr_pools.h" +// TODO (oschaaf): +// We should reparent ApacheRewriteDriverFactory and NgxRewriteDriverFactory +// to a new class OriginRewriteDriverFactory & factor out as much as possible. + namespace net_instaweb { +class AbstractSharedMem; class SlowWorker; class StaticJavaScriptManager; +class NgxServerContext; +class AprMemCache; +class NgxCache; +class NgxRewriteOptions; +class AprMemCache; +class CacheInterface; +class AsyncCache; class NgxRewriteDriverFactory : public RewriteDriverFactory { public: static const char kStaticJavaScriptPrefix[]; + static const char kMemcached[]; NgxRewriteDriverFactory(); virtual ~NgxRewriteDriverFactory(); @@ -52,13 +66,63 @@ class NgxRewriteDriverFactory : public RewriteDriverFactory { virtual void InitStaticJavascriptManager( StaticJavascriptManager* static_js_manager); + AbstractSharedMem* shared_mem_runtime() const { + return shared_mem_runtime_.get(); + } + SlowWorker* slow_worker() { return slow_worker_.get(); } - private: + // Finds a Cache for the file_cache_path in the config. If none exists, + // creates one, using all the other parameters in the ApacheConfig. + // Currently, no checking is done that the other parameters (e.g. cache + // size, cleanup interval, etc.) are consistent. + NgxCache* GetCache(NgxRewriteOptions* config); + + // Create a new AprMemCache from the given hostname[:port] specification. + AprMemCache* NewAprMemCache(const GoogleString& spec); + + // Makes a memcached-based cache if the configuration contains a + // memcached server specification. The l2_cache passed in is used + // to handle puts/gets for huge (>1M) values. NULL is returned if + // memcached is not specified for this server. + // + // If a non-null CacheInterface* is returned, its ownership is transferred + // to the caller and must be freed on destruction. + CacheInterface* GetMemcached(NgxRewriteOptions* options, CacheInterface* l2_cache); + + // Returns the filesystem metadata cache for the given config's specification + // (if it has one). NULL is returned if no cache is specified. + CacheInterface* GetFilesystemMetadataCache(NgxRewriteOptions* config); +private: SimpleStats simple_stats_; Timer* timer_; apr_pool_t* pool_; scoped_ptr slow_worker_; + scoped_ptr shared_mem_runtime_; + typedef std::map PathCacheMap; + PathCacheMap path_cache_map_; + MD5Hasher cache_hasher_; + + // memcache connections are expensive. Just allocate one per + // distinct server-list. At the moment there is no consistency + // checking for other parameters. Note that each memcached + // interface share the thread allocation, based on the + // ModPagespeedMemcachedThreads settings first encountered for + // a particular server-set. + // + // The QueuedWorkerPool for async cache-gets is shared among all + // memcached connections. + // + // The CacheInterface* value in the MemcacheMap now includes, + // depending on options, instances of CacheBatcher, AsyncCache, + // and CacheStats. Explicit lists of AprMemCache instances and + // AsyncCache objects are also included, as they require extra + // treatment during startup and shutdown. + typedef std::map MemcachedMap; + MemcachedMap memcached_map_; + scoped_ptr memcached_pool_; + std::vector memcache_servers_; + std::vector async_caches_; DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory); }; diff --git a/src/ngx_rewrite_options.cc b/src/ngx_rewrite_options.cc index 803bc7d9f..f81e27858 100644 --- a/src/ngx_rewrite_options.cc +++ b/src/ngx_rewrite_options.cc @@ -65,6 +65,12 @@ void NgxRewriteOptions::AddProperties() { add_ngx_option(1024, // 1MB &NgxRewriteOptions::lru_cache_kb_per_process_, "nlcp", RewriteOptions::kLruCacheKbPerProcess); + add_ngx_option("", &NgxRewriteOptions::memcached_servers_, "ams", + RewriteOptions::kMemcachedServers); + add_ngx_option(1, &NgxRewriteOptions::memcached_threads_, "amt", + RewriteOptions::kMemcachedThreads); + add_ngx_option(false, &NgxRewriteOptions::use_shared_mem_locking_, "ausml", + RewriteOptions::kUseSharedMemLocking); MergeSubclassProperties(ngx_properties_); NgxRewriteOptions config; diff --git a/src/ngx_rewrite_options.h b/src/ngx_rewrite_options.h index ab3db817f..5ff5f29a1 100644 --- a/src/ngx_rewrite_options.h +++ b/src/ngx_rewrite_options.h @@ -107,7 +107,24 @@ class NgxRewriteOptions : public RewriteOptions { void set_lru_cache_kb_per_process(int64 x) { set_option(x, &lru_cache_kb_per_process_); } - + bool use_shared_mem_locking() const { + return use_shared_mem_locking_.value(); + } + void set_use_shared_mem_locking(bool x) { + set_option(x, &use_shared_mem_locking_); + } + const GoogleString& memcached_servers() const { + return memcached_servers_.value(); + } + void set_memcached_servers(GoogleString x) { + set_option(x, &memcached_servers_); + } + int memcached_threads() const { + return memcached_threads_.value(); + } + void set_memcached_threads(int x) { + set_option(x, &memcached_threads_); + } private: // Used by class_name() and DynamicCast() to provide error checking. static const char kClassName[]; @@ -168,6 +185,11 @@ class NgxRewriteOptions : public RewriteOptions { Option file_cache_clean_size_kb_; Option lru_cache_byte_limit_; Option lru_cache_kb_per_process_; + Option use_shared_mem_locking_; + Option memcached_threads_; + // comma-separated list of host[:port]. See AprMemCache::AprMemCache + // for code that parses it. + Option memcached_servers_; DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions); };