caching: support memcached

Merge in Otto's #65 porting the apache memcached implementation to nginx.

README.md

@@ -165,6 +165,21 @@ and then eventually:
 
 along with a failing test because ngx_pagespeed is not yet complete.
 
+#### Testing with memcached
+
+Start an memcached server:
+
+    memcached -p 11213
+
+To the configuration above add to the main or server block:
+
+    pagespeed MemcachedServers "localhost:11213";
+    pagespeed MemcachedThreads 1;
+
+Then run the system test:
+
+    /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050
+
 ## Configuration
 
 Once configuration is complete, any mod_pagespeed configuration directive should

config

@@ -89,17 +89,23 @@ ngx_feature_test="
 if [ $ngx_found = yes ]; then
   ps_src="$ngx_addon_dir/src"
   ngx_addon_name=ngx_pagespeed
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/apr_thread_compatible_pool.cc"
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/serf_url_async_fetcher.cc"
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/apache/apr_mem_cache.cc"
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/net/instaweb/util/key_value_codec.cc"
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/../../mod_pagespeed/src/third_party/aprutil/apr_memcache2.c"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_pagespeed.cc"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_rewrite_driver_factory.cc"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_rewrite_options.cc"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_server_context.cc"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_base_fetch.cc"
+  NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ps_src/ngx_cache.cc"
   HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
   CORE_LIBS="$CORE_LIBS 
              $pagespeed_libs
-             $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/serf/libserf.a
-             $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/apr/libapr.a
-             $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/aprutil/libaprutil.a"
+             $mod_pagespeed_dir/out/Release/obj.target/third_party/serf/libserf.a
+             $mod_pagespeed_dir/out/Release/obj.target/third_party/aprutil/libaprutil.a
+             $mod_pagespeed_dir/out/Release/obj.target/third_party/apr/libapr.a"
   CORE_INCS="$CORE_INCS $pagespeed_include"
 else
   cat << END

src/ngx_cache.cc

@@ -0,0 +1,102 @@
+// Copyright 2012 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#include "ngx_cache.h"
+#include "ngx_rewrite_options.h"
+#include "ngx_rewrite_driver_factory.h"
+#include "net/instaweb/util/public/cache_interface.h"
+#include "net/instaweb/util/public/cache_stats.h"
+#include "net/instaweb/util/public/file_cache.h"
+#include "net/instaweb/util/public/file_system_lock_manager.h"
+#include "net/instaweb/util/public/lru_cache.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/shared_mem_lock_manager.h"
+#include "net/instaweb/util/public/thread_system.h"
+#include "net/instaweb/util/public/threadsafe_cache.h"
+
+namespace net_instaweb {
+
+const char NgxCache::kFileCache[] = "file_cache";
+const char NgxCache::kLruCache[] = "lru_cache";
+
+// TODO(oschaaf): refactor this to share as much as possible
+// with apache_cache.cc
+// The NgxCache shares a file cache per path, with an optional
+// LRU Cache
+NgxCache::NgxCache(const StringPiece& path,
+                   const NgxRewriteOptions& config,
+                   NgxRewriteDriverFactory* factory)
+    : path_(path.data(), path.size()),
+      factory_(factory),
+      lock_manager_(NULL),
+      file_cache_(NULL) {
+  if (config.use_shared_mem_locking()) {
+    shared_mem_lock_manager_.reset(new SharedMemLockManager(
+        factory->shared_mem_runtime(), StrCat(path, "/named_locks"),
+        factory->scheduler(), factory->hasher(), factory->message_handler()));
+    lock_manager_ = shared_mem_lock_manager_.get();
+  } else {
+    FallBackToFileBasedLocking();
+  }
+
+  FileCache::CachePolicy* policy = new FileCache::CachePolicy(
+      factory->timer(),
+      factory->hasher(),
+      config.file_cache_clean_interval_ms(),
+      config.file_cache_clean_size_kb() * 1024,
+      config.file_cache_clean_inode_limit());
+  file_cache_ = new FileCache(
+      config.file_cache_path(), factory->file_system(), NULL,
+      factory->filename_encoder(), policy, factory->message_handler());
+  l2_cache_.reset(new CacheStats(kFileCache, file_cache_, factory->timer(),
+                                 factory->statistics()));
+
+  if (config.lru_cache_kb_per_process() != 0) {
+    LRUCache* lru_cache = new LRUCache(
+        config.lru_cache_kb_per_process() * 1024);
+
+    // We only add the threadsafe-wrapper to the LRUCache.  The FileCache
+    // is naturally thread-safe because it's got no writable member variables.
+    // And surrounding that slower-running class with a mutex would likely
+    // cause contention.
+    ThreadsafeCache* ts_cache =
+        new ThreadsafeCache(lru_cache, factory->thread_system()->NewMutex());
+    // TODO(oschaaf): Non-portable (though most major compilers accept it).
+#if CACHE_STATISTICS
+    l1_cache_.reset(new CacheStats(kLruCache, ts_cache, factory->timer(),
+                                   factory->statistics()));
+#else
+    l1_cache_.reset(ts_cache);
+#endif
+  }
+}
+
+NgxCache::~NgxCache() {
+}
+
+// TODO(oschaaf): see rootinit/childinit from ApacheCache.cc
+
+void NgxCache::FallBackToFileBasedLocking() {
+  if ((shared_mem_lock_manager_.get() != NULL) || (lock_manager_ == NULL)) {
+    shared_mem_lock_manager_.reset(NULL);
+    file_system_lock_manager_.reset(new FileSystemLockManager(
+        factory_->file_system(), path_,
+        factory_->scheduler(), factory_->message_handler()));
+    lock_manager_ = file_system_lock_manager_.get();
+  }
+}
+
+}  // namespace net_instaweb

src/ngx_cache.h

@@ -0,0 +1,75 @@
+// Copyright 2012 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#ifndef NGX_CACHE_H_
+#define NGX_CACHE_H_
+
+#include "base/scoped_ptr.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+class NgxRewriteOptions;
+class NgxRewriteDriverFactory;
+class CacheInterface;
+class FileCache;
+class FileSystemLockManager;
+class MessageHandler;
+class NamedLockManager;
+class SharedMemLockManager;
+
+// The NgxCache encapsulates a cache-sharing model where a user specifies
+// a file-cache path per virtual-host.  With each file-cache object we keep
+// a locking mechanism and an optional per-process LRUCache.
+class NgxCache {
+ public:
+  static const char kFileCache[];
+  static const char kLruCache[];
+
+  NgxCache(const StringPiece& path,
+              const NgxRewriteOptions& config,
+              NgxRewriteDriverFactory* factory);
+  ~NgxCache();
+  CacheInterface* l1_cache() { return l1_cache_.get(); }
+  CacheInterface* l2_cache() { return l2_cache_.get(); }
+  NamedLockManager* lock_manager() { return lock_manager_; }
+
+  void RootInit();
+  void ChildInit();
+  void GlobalCleanup(MessageHandler* handler);  // only called in root process
+
+ private:
+  void FallBackToFileBasedLocking();
+
+  GoogleString path_;
+  NgxRewriteDriverFactory* factory_;
+  scoped_ptr<SharedMemLockManager> shared_mem_lock_manager_;
+  scoped_ptr<FileSystemLockManager> file_system_lock_manager_;
+  NamedLockManager* lock_manager_;
+  FileCache* file_cache_;  // owned by l2 cache
+  scoped_ptr<CacheInterface> l1_cache_;
+  scoped_ptr<CacheInterface> l2_cache_;
+};
+
+// CACHE_STATISTICS is #ifdef'd to facilitate experiments with whether
+// tracking the detailed stats & histograms has a QPS impact.  Set it
+// to 0 to turn it off.
+#define CACHE_STATISTICS 1
+
+}  // namespace net_instaweb
+
+#endif  // NGX_CACHE_H_

src/ngx_pagespeed.cc

@@ -280,6 +280,9 @@ ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char*
 ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+void
+ps_ignore_sigpipe();
+
 ngx_command_t ps_commands[] = {
   { ngx_string("pagespeed"),
     NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|
@@ -300,6 +303,16 @@ ngx_command_t ps_commands[] = {
   ngx_null_command
 };
 
+void
+ps_ignore_sigpipe() {
+  struct sigaction act;
+  ngx_memzero(&act, sizeof(act));
+  act.sa_handler = SIG_IGN;
+  sigemptyset (&act.sa_mask);
+  act.sa_flags = 0;
+  sigaction (SIGPIPE, &act, NULL);
+}
+
 #define NGX_PAGESPEED_MAX_ARGS 10
 char*
 ps_configure(ngx_conf_t* cf,
@@ -430,6 +443,11 @@ ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
   // because if there are no server blocks with pagespeed configuration
   // directives then we don't want it initialized.
   if (cfg_m->driver_factory == NULL) {
+    // TODO(oschaaf): this ignores sigpipe messages from memcached.
+    // however, it would be better to not have those signals generated
+    // in the first place, as suppressing them this way may interfere
+    // with other modules that actually are interested in these signals
+    ps_ignore_sigpipe();
     net_instaweb::NgxRewriteDriverFactory::Initialize();
     // TODO(jefftk): We should call NgxRewriteDriverFactory::Terminate() when
     // we're done with it.  That never happens, though, because this is the

src/ngx_rewrite_driver_factory.cc

@@ -44,10 +44,19 @@
 #include "net/instaweb/util/public/write_through_cache.h"
 #include "net/instaweb/http/public/http_cache.h"
 #include "net/instaweb/http/public/write_through_http_cache.h"
-
-// TODO(oschaaf): discuss the proper way to to this.
-#include "net/instaweb/apache/apr_thread_compatible_pool.cc"
-#include "net/instaweb/apache/serf_url_async_fetcher.cc"
+#include "net/instaweb/apache/apr_thread_compatible_pool.h"
+#include "net/instaweb/apache/serf_url_async_fetcher.h"
+#include "net/instaweb/apache/apr_mem_cache.h"
+#include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/cache_copy.h"
+#include "net/instaweb/util/public/async_cache.h"
+#include "net/instaweb/util/public/cache_stats.h"
+#include "net/instaweb/util/public/cache_batcher.h"
+#include "net/instaweb/util/public/fallback_cache.h"
+#include "ngx_cache.h"
+#include "net/instaweb/apache/apr_thread_compatible_pool.h"
+#include "net/instaweb/apache/serf_url_async_fetcher.h"
+#include "net/instaweb/apache/apr_mem_cache.h"
 
 namespace net_instaweb {
 
@@ -61,20 +70,42 @@ class UrlAsyncFetcher;
 class UrlFetcher;
 class Writer;
 
-NgxRewriteDriverFactory::NgxRewriteDriverFactory() {
+const char NgxRewriteDriverFactory::kMemcached[] = "memcached";
+
+NgxRewriteDriverFactory::NgxRewriteDriverFactory() :
+  shared_mem_runtime_(new NullSharedMem()),
+  cache_hasher_(20) {
   RewriteDriverFactory::InitStats(&simple_stats_);
   SerfUrlAsyncFetcher::InitStats(&simple_stats_);
+  AprMemCache::InitStats(&simple_stats_);
+  CacheStats::InitStats(NgxCache::kFileCache, &simple_stats_);
+  CacheStats::InitStats(NgxCache::kLruCache, &simple_stats_);
+  CacheStats::InitStats(kMemcached, &simple_stats_);
   SetStatistics(&simple_stats_);
   timer_ = DefaultTimer();
   apr_initialize();
   apr_pool_create(&pool_,NULL);
-
   InitializeDefaultOptions();
 }
 
 NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
   delete timer_;
+  timer_ = NULL;
   slow_worker_->ShutDown();
+  apr_pool_destroy(pool_);
+  pool_ = NULL;
+
+  for (PathCacheMap::iterator p = path_cache_map_.begin(),
+           e = path_cache_map_.end(); p != e; ++p) {
+    NgxCache* cache = p->second;
+    defer_cleanup(new Deleter<NgxCache>(cache));
+  }
+
+  for (MemcachedMap::iterator p = memcached_map_.begin(),
+           e = memcached_map_.end(); p != e; ++p) {
+    CacheInterface* memcached = p->second;
+    defer_cleanup(new Deleter<CacheInterface>(memcached));
+  }
 }
 
 const char NgxRewriteDriverFactory::kStaticJavaScriptPrefix[] =
@@ -128,35 +159,45 @@ void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
   NgxRewriteOptions* options = NgxRewriteOptions::DynamicCast(
       server_context->global_options());
 
-  LRUCache* lru_cache = new LRUCache(
-      options->lru_cache_kb_per_process() * 1024);
-  CacheInterface* cache = new ThreadsafeCache(
-      lru_cache, thread_system()->NewMutex());
+  NgxCache* ngx_cache = GetCache(options);
+  CacheInterface* l1_cache = ngx_cache->l1_cache();
+  CacheInterface* l2_cache = ngx_cache->l2_cache();
+  CacheInterface* memcached = GetMemcached(options, l2_cache);
+  if (memcached != NULL) {
+    // XXX(oschaaf): remove when done
+    l1_cache = NULL;
+    l2_cache = memcached;
+    server_context->set_owned_cache(memcached);
+    server_context->set_filesystem_metadata_cache(
+        new CacheCopy(GetFilesystemMetadataCache(options)));
+  }
+  Statistics* stats = server_context->statistics();
 
-  FileCache::CachePolicy* policy = new FileCache::CachePolicy(
-      timer(),
-      hasher(),
-      options->file_cache_clean_interval_ms(),
-      options->file_cache_clean_size_kb() * 1024,
-      options->file_cache_clean_inode_limit());
+  // TODO(jmarantz): consider moving ownership of the L1 cache into the
+  // factory, rather than having one per vhost.
+  //
+  // Note that a user can disable the L1 cache by setting its byte-count
+  // to 0, in which case we don't build the write-through mechanisms.
+  if (l1_cache == NULL) {
+    HTTPCache* http_cache = new HTTPCache(l2_cache, timer(), hasher(), stats);
+    server_context->set_http_cache(http_cache);
+    server_context->set_metadata_cache(new CacheCopy(l2_cache));
+    server_context->MakePropertyCaches(l2_cache);
+  } else {
+    WriteThroughHTTPCache* write_through_http_cache = new WriteThroughHTTPCache(
+        l1_cache, l2_cache, timer(), hasher(), stats);
+    write_through_http_cache->set_cache1_limit(options->lru_cache_byte_limit());
+    server_context->set_http_cache(write_through_http_cache);
 
-  FileCache* file_cache = new FileCache(options->file_cache_path(),
-                                        file_system(), NULL,
-                                        filename_encoder(), policy,
-                                        message_handler());
+    WriteThroughCache* write_through_cache = new WriteThroughCache(
+        l1_cache, l2_cache);
+    write_through_cache->set_cache1_limit(options->lru_cache_byte_limit());
+    server_context->set_metadata_cache(write_through_cache);
+    server_context->MakePropertyCaches(l2_cache);
+  }
 
-  slow_worker_.reset(new SlowWorker(thread_system()));
-
-  WriteThroughHTTPCache* write_through_http_cache = new WriteThroughHTTPCache(
-      cache, file_cache, timer(), hasher(), statistics());
-  write_through_http_cache->set_cache1_limit(options->lru_cache_byte_limit());
-  server_context->set_http_cache(write_through_http_cache);
-
-  WriteThroughCache* write_through_cache = new WriteThroughCache(
-      cache, file_cache);
-  write_through_cache->set_cache1_limit(options->lru_cache_byte_limit());
-  server_context->set_metadata_cache(write_through_cache);
-  server_context->MakePropertyCaches(file_cache);
+  // TODO(oschaaf): see the property cache setup in the apache rewrite
+  // driver factory
   server_context->set_enable_property_cache(true);
 }
 
@@ -173,4 +214,114 @@ void NgxRewriteDriverFactory::InitStaticJavascriptManager(
   static_js_manager->set_library_url_prefix(kStaticJavaScriptPrefix);
 }
 
+NgxCache* NgxRewriteDriverFactory::GetCache(NgxRewriteOptions* options) {
+  const GoogleString& path = options->file_cache_path();
+  std::pair<PathCacheMap::iterator, bool> result = path_cache_map_.insert(
+      PathCacheMap::value_type(path, static_cast<NgxCache*>(NULL)));
+  PathCacheMap::iterator iter = result.first;
+  if (result.second) {
+    iter->second = new NgxCache(path, *options, this);
+  }
+  return iter->second;
+}
+
+AprMemCache* NgxRewriteDriverFactory::NewAprMemCache(
+    const GoogleString& spec) {
+  // TODO(oschaaf): determine a sensible limit
+  int thread_limit = 2;
+  return new AprMemCache(spec, thread_limit, &cache_hasher_, statistics(),
+                         timer(), message_handler());
+}
+
+CacheInterface* NgxRewriteDriverFactory::GetMemcached(
+    NgxRewriteOptions* options, CacheInterface* l2_cache) {
+  CacheInterface* memcached = NULL;
+
+  // Find a memcache that matches the current spec, or create a new one
+  // if needed. Note that this means that two different VirtualHost's will
+  // share a memcached if their specs are the same but will create their own
+  // if the specs are different.
+  if (!options->memcached_servers().empty()) {
+    const GoogleString& server_spec = options->memcached_servers();
+    std::pair<MemcachedMap::iterator, bool> result = memcached_map_.insert(
+        MemcachedMap::value_type(server_spec, memcached));
+    if (result.second) {
+      fprintf(stderr,"setting up memcached server [%s]\n",server_spec.c_str());
+      AprMemCache* mem_cache = NewAprMemCache(server_spec);
+
+      memcache_servers_.push_back(mem_cache);
+
+      int num_threads = options->memcached_threads();
+      if (num_threads != 0) {
+        if (memcached_pool_.get() == NULL) {
+          // Note -- we will use the first value of ModPagespeedMemCacheThreads
+          // that we see in a VirtualHost, ignoring later ones.
+          memcached_pool_.reset(new QueuedWorkerPool(num_threads,
+                                                     thread_system()));
+        }
+        AsyncCache* async_cache = new AsyncCache(mem_cache,
+                                                 memcached_pool_.get());
+        async_caches_.push_back(async_cache);
+        memcached = async_cache;
+      } else {
+        message_handler()->Message(kWarning,
+          "Running memcached synchronously, this may hurt performance");
+        memcached = mem_cache;
+      }
+
+      // Put the batcher above the stats so that the stats sees the MultiGets
+      // and can show us the histogram of how they are sized.
+#if CACHE_STATISTICS
+      memcached = new CacheStats(kMemcached, memcached, timer(), statistics());
+#endif
+      CacheBatcher* batcher = new CacheBatcher(
+          memcached, thread_system()->NewMutex(), statistics());
+      if (num_threads != 0) {
+        batcher->set_max_parallel_lookups(num_threads);
+      }
+      memcached = batcher;
+      result.first->second = memcached;
+
+      // TODO(oschaaf): should not connect to memcached here
+      bool connected = mem_cache->Connect();
+      fprintf(stderr,
+              "connected to memcached backend: %s\n", connected ? "true": "false");
+
+    } else {
+      memcached = result.first->second;
+    }
+
+    // Note that a distinct FallbackCache gets created for every VirtualHost
+    // that employs memcached, even if the memcached and file-cache
+    // specifications are identical.  This does no harm, because there
+    // is no data in the cache object itself; just configuration.  Sharing
+    // FallbackCache* objects would require making a map using the
+    // memcache & file-cache specs as a key, so it's simpler to make a new
+    // small FallbackCache object for each VirtualHost.
+    memcached = new FallbackCache(memcached, l2_cache,
+                                  AprMemCache::kValueSizeThreshold,
+                                  message_handler());
+  }
+  return memcached;
+}
+
+CacheInterface* NgxRewriteDriverFactory::GetFilesystemMetadataCache(
+    NgxRewriteOptions* options) {
+  // Reuse the memcached server(s) for the filesystem metadata cache. We need
+  // to search for our config's entry in the vector of servers (not the more
+  // obvious map) because the map's entries are wrapped in an AsyncCache, and
+  // the filesystem metadata cache requires a blocking cache (like memcached).
+  // Note that if we have a server spec we *know* it's in the searched vector.
+  // We perform a linear scan assuming that the searched set will be small
+  DCHECK_EQ(options->memcached_servers().empty(), memcache_servers_.empty());
+  const GoogleString& server_spec = options->memcached_servers();
+  for (int i = 0, n = memcache_servers_.size(); i < n; ++i) {
+    if (server_spec == memcache_servers_[i]->server_spec()) {
+      return memcache_servers_[i];
+    }
+  }
+
+  return NULL;
+}
+
 }  // namespace net_instaweb

src/ngx_rewrite_driver_factory.h

@@ -21,17 +21,31 @@
 
 #include "base/scoped_ptr.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
+#include "net/instaweb/util/public/md5_hasher.h"
 #include "net/instaweb/util/public/simple_stats.h"
 #include "apr_pools.h"
 
+// TODO (oschaaf):
+// We should reparent ApacheRewriteDriverFactory and NgxRewriteDriverFactory
+// to a new class OriginRewriteDriverFactory & factor out as much as possible.
+
 namespace net_instaweb {
 
+class AbstractSharedMem;
 class SlowWorker;
 class StaticJavaScriptManager;
+class NgxServerContext;
+class AprMemCache;
+class NgxCache;
+class NgxRewriteOptions;
+class AprMemCache;
+class CacheInterface;
+class AsyncCache;
 
 class NgxRewriteDriverFactory : public RewriteDriverFactory {
  public:
   static const char kStaticJavaScriptPrefix[];
+  static const char kMemcached[];
 
   NgxRewriteDriverFactory();
   virtual ~NgxRewriteDriverFactory();
@@ -52,13 +66,63 @@ class NgxRewriteDriverFactory : public RewriteDriverFactory {
   virtual void InitStaticJavascriptManager(
       StaticJavascriptManager* static_js_manager);
 
+  AbstractSharedMem* shared_mem_runtime() const {
+    return shared_mem_runtime_.get();
+  }
+
   SlowWorker* slow_worker() { return slow_worker_.get(); }
 
- private:
+  // Finds a Cache for the file_cache_path in the config.  If none exists,
+  // creates one, using all the other parameters in the ApacheConfig.
+  // Currently, no checking is done that the other parameters (e.g. cache
+  // size, cleanup interval, etc.) are consistent.
+  NgxCache* GetCache(NgxRewriteOptions* config);
+
+  // Create a new AprMemCache from the given hostname[:port] specification.
+  AprMemCache* NewAprMemCache(const GoogleString& spec);
+
+  // Makes a memcached-based cache if the configuration contains a
+  // memcached server specification.  The l2_cache passed in is used
+  // to handle puts/gets for huge (>1M) values.  NULL is returned if
+  // memcached is not specified for this server.
+  //
+  // If a non-null CacheInterface* is returned, its ownership is transferred
+  // to the caller and must be freed on destruction.
+  CacheInterface* GetMemcached(NgxRewriteOptions* options, CacheInterface* l2_cache);
+
+  // Returns the filesystem metadata cache for the given config's specification
+  // (if it has one). NULL is returned if no cache is specified.
+  CacheInterface* GetFilesystemMetadataCache(NgxRewriteOptions* config);
+private:
   SimpleStats simple_stats_;
   Timer* timer_;
   apr_pool_t* pool_;
   scoped_ptr<SlowWorker> slow_worker_;
+  scoped_ptr<AbstractSharedMem> shared_mem_runtime_;
+  typedef std::map<GoogleString, NgxCache*> PathCacheMap;
+  PathCacheMap path_cache_map_;
+  MD5Hasher cache_hasher_;
+
+  // memcache connections are expensive.  Just allocate one per
+  // distinct server-list.  At the moment there is no consistency
+  // checking for other parameters.  Note that each memcached
+  // interface share the thread allocation, based on the
+  // ModPagespeedMemcachedThreads settings first encountered for
+  // a particular server-set.
+  //
+  // The QueuedWorkerPool for async cache-gets is shared among all
+  // memcached connections.
+  //
+  // The CacheInterface* value in the MemcacheMap now includes,
+  // depending on options, instances of CacheBatcher, AsyncCache,
+  // and CacheStats.  Explicit lists of AprMemCache instances and
+  // AsyncCache objects are also included, as they require extra
+  // treatment during startup and shutdown.
+  typedef std::map<GoogleString, CacheInterface*> MemcachedMap;
+  MemcachedMap memcached_map_;
+  scoped_ptr<QueuedWorkerPool> memcached_pool_;
+  std::vector<AprMemCache*> memcache_servers_;
+  std::vector<AsyncCache*> async_caches_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

src/ngx_rewrite_options.cc

@@ -65,6 +65,12 @@ void NgxRewriteOptions::AddProperties() {
   add_ngx_option(1024,  // 1MB
                  &NgxRewriteOptions::lru_cache_kb_per_process_, "nlcp",
                  RewriteOptions::kLruCacheKbPerProcess);
+  add_ngx_option("", &NgxRewriteOptions::memcached_servers_, "ams",
+                 RewriteOptions::kMemcachedServers);
+  add_ngx_option(1, &NgxRewriteOptions::memcached_threads_, "amt",
+                 RewriteOptions::kMemcachedThreads);
+  add_ngx_option(false, &NgxRewriteOptions::use_shared_mem_locking_, "ausml",
+                 RewriteOptions::kUseSharedMemLocking);
 
   MergeSubclassProperties(ngx_properties_);
   NgxRewriteOptions config;

src/ngx_rewrite_options.h

@@ -107,7 +107,24 @@ class NgxRewriteOptions : public RewriteOptions {
   void set_lru_cache_kb_per_process(int64 x) {
     set_option(x, &lru_cache_kb_per_process_);
   }
-
+  bool use_shared_mem_locking() const {
+    return use_shared_mem_locking_.value();
+  }
+  void set_use_shared_mem_locking(bool x) {
+    set_option(x, &use_shared_mem_locking_);
+  }
+  const GoogleString& memcached_servers() const {
+    return memcached_servers_.value();
+  }
+  void set_memcached_servers(GoogleString x) {
+    set_option(x, &memcached_servers_);
+  }
+  int memcached_threads() const {
+    return memcached_threads_.value();
+  }
+  void set_memcached_threads(int x) {
+    set_option(x, &memcached_threads_);
+  }
  private:
   // Used by class_name() and DynamicCast() to provide error checking.
   static const char kClassName[];
@@ -168,6 +185,11 @@ class NgxRewriteOptions : public RewriteOptions {
   Option<int64> file_cache_clean_size_kb_;
   Option<int64> lru_cache_byte_limit_;
   Option<int64> lru_cache_kb_per_process_;
+  Option<bool> use_shared_mem_locking_;
+  Option<int> memcached_threads_;
+  // comma-separated list of host[:port].  See AprMemCache::AprMemCache
+  // for code that parses it.
+  Option<GoogleString> memcached_servers_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
 };