artox/incubator-pagespeed-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #469 from pagespeed/jefftk-update-trunk-tracking

Browse Source 
trunk-tracking: take advantage of new functionality in system/
This commit is contained in:
Jeff Kaufman
2013-08-05 14:24:25 -07:00
parent 9b3ab25cea 51eaeddfba
commit bb1093e8b2
11 changed files with 94 additions and 720 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config
+1 -9
View File
@@ -156,8 +156,7 @@ if [ $ngx_found = yes ]; then
$ps_src/ngx_rewrite_options.h \
$ps_src/ngx_server_context.h \
$ps_src/ngx_thread_system.h \
$ps_src/ngx_url_async_fetcher.h \
$ps_src/pthread_shared_mem.h"
$ps_src/ngx_url_async_fetcher.h"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
$ps_src/log_message_handler.cc \
$ps_src/ngx_base_fetch.cc \
@@ -172,19 +171,12 @@ if [ $ngx_found = yes ]; then
$ps_src/ngx_server_context.cc \
$ps_src/ngx_thread_system.cc \
$ps_src/ngx_url_async_fetcher.cc \
$ps_src/pthread_shared_mem.cc \
$mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/system/console_out.cc \
$mod_pagespeed_dir/out/$buildtype/obj/gen/data2c_out/instaweb/net/instaweb/system/console_css_out.cc \
$mod_pagespeed_dir/net/instaweb/system/add_headers_fetcher.cc \
$ps_src/in_place_resource_recorder.cc \
$mod_pagespeed_dir/net/instaweb/system/loopback_route_fetcher.cc \
$mod_pagespeed_dir/net/instaweb/system/serf_url_async_fetcher.cc"
# TODO(jefftk): when the next release of PSOL after 1.6.29.3 comes out, change
# $ps_src/in_place_resource_recorder.cc
# above to
# $mod_pagespeed_dir/net/instaweb/apache/in_place_resource_recorder.cc
# Make pagespeed run immediately before gzip.
HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
scripts/prepare_psol.sh
-1
View File
@@ -70,7 +70,6 @@ rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
--include="sparse_hash_map" \
--include="sparse_hash_set" \
--include="sparsetable" \
--include="in_place_resource_recorder.cc" \
--exclude='*'
mkdir -p psol/lib/Debug/linux/ia32
mkdir -p psol/lib/Debug/linux/x64
src/in_place_resource_recorder.cc
-114
View File
@@ -1,114 +0,0 @@
// Copyright 2013 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: sligocki@google.com (Shawn Ligocki)
// Copied from net/instaweb/apache/in_place_resource_recorder.cc
// The next release of PSOL will include that file, at which point this file can
// be removed.
// TODO(jefftk): delete this when the next PSOL release after 1.6.29.3 is ready.
#include "net/instaweb/apache/in_place_resource_recorder.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/http_value.h"
#include "net/instaweb/http/public/meta_data.h"
#include "net/instaweb/http/public/request_headers.h"
#include "net/instaweb/http/public/response_headers.h"
#include "net/instaweb/util/public/statistics.h"
#include "net/instaweb/util/public/string_util.h"
namespace net_instaweb {
class MessageHandler;
namespace {
const char kNumResources[] = "ipro_recorder_resources";
const char kNumInsertedIntoCache[] = "ipro_recorder_inserted_into_cache";
const char kNumNotCacheable[] = "ipro_recorder_not_cacheable";
const char kNumFailed[] = "ipro_recorder_failed";
}
InPlaceResourceRecorder::InPlaceResourceRecorder(
StringPiece url, RequestHeaders* request_headers, bool respect_vary,
HTTPCache* cache, Statistics* stats, MessageHandler* handler)
: url_(url.data(), url.size()), request_headers_(request_headers),
respect_vary_(respect_vary), success_(true), cache_(cache),
handler_(handler),
num_resources_(stats->GetVariable(kNumResources)),
num_inserted_into_cache_(stats->GetVariable(kNumInsertedIntoCache)),
num_not_cacheable_(stats->GetVariable(kNumNotCacheable)),
num_failed_(stats->GetVariable(kNumFailed)) {
num_resources_->Add(1);
}
InPlaceResourceRecorder::~InPlaceResourceRecorder() {
}
void InPlaceResourceRecorder::InitStats(Statistics* statistics) {
statistics->AddVariable(kNumResources);
statistics->AddVariable(kNumInsertedIntoCache);
statistics->AddVariable(kNumNotCacheable);
statistics->AddVariable(kNumFailed);
}
bool InPlaceResourceRecorder::Write(const StringPiece& contents,
MessageHandler* handler) {
return resource_value_.Write(contents, handler);
}
bool InPlaceResourceRecorder::Flush(MessageHandler* handler) {
return true;
}
void InPlaceResourceRecorder::DoneAndSetHeaders(
ResponseHeaders* response_headers) {
if (success_) {
// TODO(sligocki): Currently this will cache HTML and other Content-Types
// that cannot be rewritten in-place. It seems like we should only cache
// resources that have a hope of being rewritten in-place, and store a
// failure in the IPRO meta-data lookup for all others.
bool is_cacheable =
response_headers->IsProxyCacheableGivenRequest(*request_headers_);
if (is_cacheable && respect_vary_) {
is_cacheable = response_headers->VaryCacheable(
request_headers_->Has(HttpAttributes::kCookie));
}
// Put resource_value_ in cache.
if (is_cacheable) {
resource_value_.SetHeaders(response_headers);
cache_->Put(url_, &resource_value_, handler_);
// TODO(sligocki): Start IPRO rewrite.
num_inserted_into_cache_->Add(1);
} else {
cache_->RememberNotCacheable(url_, response_headers->status_code() == 200,
handler_);
num_not_cacheable_->Add(1);
}
} else {
// TODO(sligocki): Should we RememberFetchFailed() if success_ == false?
// We don't expect this to happen much, it seems to only happen with Apache
// Bucket Brigade issues.
num_failed_->Add(1);
}
delete this;
}
} // namespace net_instaweb
src/ngx_pagespeed.cc
+32 -13
View File
@@ -39,10 +39,9 @@
#include "ngx_thread_system.h"
#include "apr_time.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/apache/in_place_resource_recorder.h"
#include "net/instaweb/automatic/public/proxy_fetch.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/rewriter/public/experiment_matcher.h"
@@ -50,15 +49,17 @@
#include "net/instaweb/rewriter/public/process_context.h"
#include "net/instaweb/rewriter/public/resource_fetch.h"
#include "net/instaweb/rewriter/public/rewrite_driver.h"
#include "net/instaweb/rewriter/public/rewrite_stats.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/system/public/handlers.h"
#include "net/instaweb/system/public/in_place_resource_recorder.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/public/global_constants.h"
#include "net/instaweb/public/version.h"
#include "net/instaweb/util/public/fallback_property_page.h"
#include "net/instaweb/util/public/google_message_handler.h"
#include "net/instaweb/util/public/google_url.h"
#include "net/instaweb/util/public/gzip_inflater.h"
#include "pthread_shared_mem.h"
#include "net/instaweb/util/public/query_params.h"
#include "net/instaweb/util/public/statistics_logger.h"
#include "net/instaweb/util/public/stdio_file_system.h"
@@ -66,7 +67,7 @@
#include "net/instaweb/util/public/string_writer.h"
#include "net/instaweb/util/public/time_util.h"
#include "net/instaweb/util/stack_buffer.h"
#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "pagespeed/kernel/thread/pthread_shared_mem.h"
extern ngx_module_t ngx_pagespeed;
@@ -82,6 +83,9 @@ extern ngx_module_t ngx_pagespeed;
// http://lxr.evanmiller.org/http/source/http/ngx_http_request.h#L130
#define NGX_HTTP_PAGESPEED_BUFFERED 0x08
// Needed for SystemRewriteDriverFactory to use shared memory.
#define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
namespace ngx_psol {
const char* kInternalEtagName = "@psol-etag";
@@ -702,7 +706,9 @@ void* ps_create_main_conf(ngx_conf_t* cf) {
net_instaweb::NgxRewriteDriverFactory::Initialize();
cfg_m->driver_factory = new net_instaweb::NgxRewriteDriverFactory(
new net_instaweb::NgxThreadSystem());
new net_instaweb::NgxThreadSystem(),
"" /* hostname, not used */,
-1 /* port, not used */);
ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
return cfg_m;
}
@@ -2597,12 +2603,23 @@ ngx_int_t ps_statistics_handler(
writer.Write("</pre>", message_handler);
statistics->RenderHistograms(&writer, message_handler);
int flags = net_instaweb::SystemCaches::kDefaultStatFlags;
if (global_stats_request) {
flags |= net_instaweb::SystemCaches::kGlobalView;
}
if (params.Has("memcached")) {
GoogleString memcached_stats;
factory->PrintMemCacheStats(&memcached_stats);
if (!memcached_stats.empty()) {
ps_write_pre(memcached_stats, &writer, message_handler);
}
flags |= net_instaweb::SystemCaches::kIncludeMemcached;
}
GoogleString backend_stats;
factory->caches()->PrintCacheStats(
static_cast<net_instaweb::SystemCaches::StatFlags>(flags),
&backend_stats);
if (!backend_stats.empty()) {
writer.Write("<pre>\n", message_handler);
writer.Write(backend_stats, message_handler);
writer.Write("</pre>\n", message_handler);
}
}
@@ -3039,7 +3056,7 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
// allows statistics to work if ngx_pagespeed gets turned on via
// .htaccess or query param.
if ((statistics == NULL) && config->statistics_enabled()) {
statistics = \
statistics =
cfg_m->driver_factory->MakeGlobalSharedMemStatistics(*config);
}
@@ -3085,7 +3102,8 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
return NGX_ERROR;
}
cfg_m->driver_factory->RootInit(cycle->log);
cfg_m->driver_factory->LoggingInit(cycle->log);
cfg_m->driver_factory->RootInit();
} else {
delete cfg_m->driver_factory;
cfg_m->driver_factory = NULL;
@@ -3104,7 +3122,8 @@ ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
// ChildInit() will initialise all ServerContexts, which we need to
// create ProxyFetchFactories below
cfg_m->driver_factory->ChildInit(cycle->log);
cfg_m->driver_factory->LoggingInit(cycle->log);
cfg_m->driver_factory->ChildInit();
ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
src/ngx_rewrite_driver_factory.cc
+16 -167
View File
@@ -26,7 +26,6 @@
#include "ngx_server_context.h"
#include "ngx_thread_system.h"
#include "ngx_url_async_fetcher.h"
#include "pthread_shared_mem.h"
#include "net/instaweb/http/public/content_type.h"
#include "net/instaweb/http/public/rate_controller.h"
@@ -36,7 +35,7 @@
#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "net/instaweb/rewriter/public/static_asset_manager.h"
#include "net/instaweb/apache/in_place_resource_recorder.h"
#include "net/instaweb/system/public/in_place_resource_recorder.h"
#include "net/instaweb/system/public/serf_url_async_fetcher.h"
#include "net/instaweb/system/public/system_caches.h"
#include "net/instaweb/system/public/system_rewrite_options.h"
@@ -52,6 +51,7 @@
#include "net/instaweb/util/public/string.h"
#include "net/instaweb/util/public/string_util.h"
#include "net/instaweb/util/public/thread_system.h"
#include "pagespeed/kernel/thread/pthread_shared_mem.h"
namespace net_instaweb {
@@ -67,33 +67,24 @@ class Writer;
const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
"/ngx_pagespeed_static/";
namespace {
const char kShutdownCount[] = "child_shutdown_count";
} // namespace
class SharedCircularBuffer;
NgxRewriteDriverFactory::NgxRewriteDriverFactory(
NgxThreadSystem* ngx_thread_system)
: SystemRewriteDriverFactory(ngx_thread_system),
NgxThreadSystem* ngx_thread_system, StringPiece hostname, int port)
: SystemRewriteDriverFactory(ngx_thread_system, hostname, port),
ngx_thread_system_(ngx_thread_system),
// TODO(oschaaf): mod_pagespeed ifdefs this:
shared_mem_runtime_(new ngx::PthreadSharedMem()),
main_conf_(NULL),
threads_started_(false),
use_per_vhost_statistics_(false),
is_root_process_(true),
ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
ngx_html_parse_message_handler_(
new NgxMessageHandler(thread_system()->NewMutex())),
install_crash_handler_(false),
message_buffer_size_(0),
shared_circular_buffer_(NULL),
statistics_frozen_(false),
ngx_url_async_fetcher_(NULL),
log_(NULL),
resolver_timeout_(NGX_CONF_UNSET_MSEC),
use_native_fetcher_(false) {
use_native_fetcher_(false),
ngx_shared_circular_buffer_(NULL) {
InitializeDefaultOptions();
default_options()->set_beacon_url("/ngx_pagespeed_beacon");
SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -102,19 +93,12 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
system_options->set_avoid_renaming_introspective_javascript(true);
set_message_handler(ngx_message_handler_);
set_html_parse_message_handler(ngx_html_parse_message_handler_);
// see https://code.google.com/p/modpagespeed/issues/detail?id=672
int thread_limit = 1;
caches_.reset(
new SystemCaches(this, shared_mem_runtime_.get(), thread_limit));
}
NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
ShutDown();
CHECK(uninitialized_server_contexts_.empty() || is_root_process_);
ngx_shared_circular_buffer_ = NULL;
STLDeleteElements(&uninitialized_server_contexts_);
shared_mem_statistics_.reset(NULL);
}
Hasher* NgxRewriteDriverFactory::NewHasher() {
@@ -201,20 +185,6 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
return NULL;
}
void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
// TODO(anupama): Remove duplication wrt mod_pagespeed code.
caches_->SetupCaches(server_context);
server_context->set_enable_property_cache(true);
PropertyCache* pcache = server_context->page_property_cache();
const PropertyCache::Cohort* cohort =
server_context->AddCohort(RewriteDriver::kBeaconCohort, pcache);
server_context->set_beacon_cohort(cohort);
cohort = server_context->AddCohort(RewriteDriver::kDomCohort, pcache);
server_context->set_dom_cohort(cohort);
}
RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
options->SetRewriteLevel(RewriteOptions::kCoreFilters);
@@ -227,12 +197,6 @@ void NgxRewriteDriverFactory::InitStaticAssetManager(
static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
}
void NgxRewriteDriverFactory::PrintMemCacheStats(GoogleString* out) {
// TODO(morlovich): Port the client code to proper API, so it gets
// shm stats, too.
caches_->PrintCacheStats(SystemCaches::kIncludeMemcached, out);
}
bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetcher() {
if (ngx_url_async_fetcher_ == NULL) {
return true;
@@ -248,11 +212,6 @@ bool NgxRewriteDriverFactory::CheckResolver() {
return true;
}
void NgxRewriteDriverFactory::StopCacheActivity() {
RewriteDriverFactory::StopCacheActivity();
caches_->StopCacheActivity();
}
NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext() {
NgxServerContext* server_context = new NgxServerContext(this);
uninitialized_server_contexts_.insert(server_context);
@@ -264,13 +223,7 @@ ServerContext* NgxRewriteDriverFactory::NewServerContext() {
return NULL;
}
void NgxRewriteDriverFactory::ShutDown() {
StopCacheActivity();
if (!is_root_process_) {
Variable* child_shutdown_count = statistics()->GetVariable(kShutdownCount);
child_shutdown_count->Add(1);
}
void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
ngx_message_handler_->set_buffer(NULL);
ngx_html_parse_message_handler_->set_buffer(NULL);
for (NgxMessageHandlerSet::iterator p =
@@ -279,20 +232,6 @@ void NgxRewriteDriverFactory::ShutDown() {
(*p)->set_buffer(NULL);
}
server_context_message_handlers_.clear();
RewriteDriverFactory::ShutDown();
caches_->ShutDown(message_handler());
if (is_root_process_) {
// Cleanup statistics.
// TODO(morlovich): This looks dangerous with async.
if (shared_mem_statistics_.get() != NULL) {
shared_mem_statistics_->GlobalCleanup(message_handler());
}
if (shared_circular_buffer_ != NULL) {
shared_circular_buffer_->GlobalCleanup(message_handler());
}
}
}
void NgxRewriteDriverFactory::StartThreads() {
@@ -309,107 +248,17 @@ void NgxRewriteDriverFactory::StartThreads() {
threads_started_ = true;
}
void NgxRewriteDriverFactory::ParentOrChildInit(ngx_log_t* log) {
void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
if (install_crash_handler_) {
NgxMessageHandler::InstallCrashHandler(log);
}
ngx_message_handler_->set_log(log);
ngx_html_parse_message_handler_->set_log(log);
SharedCircularBufferInit(is_root_process_);
}
// TODO(jmarantz): make this per-vhost.
void NgxRewriteDriverFactory::SharedCircularBufferInit(bool is_root) {
// Set buffer size to 0 means turning it off
if (shared_mem_runtime() != NULL && (message_buffer_size_ != 0)) {
// TODO(jmarantz): it appears that filename_prefix() is not actually
// established at the time of this construction, calling into question
// whether we are naming our shared-memory segments correctly.
shared_circular_buffer_.reset(new SharedCircularBuffer(
shared_mem_runtime(),
message_buffer_size_,
filename_prefix().as_string(),
"foo.com" /*hostname_identifier()*/));
if (shared_circular_buffer_->InitSegment(is_root, message_handler())) {
ngx_message_handler_->set_buffer(shared_circular_buffer_.get());
ngx_html_parse_message_handler_->set_buffer(
shared_circular_buffer_.get());
}
}
}
void NgxRewriteDriverFactory::RootInit(ngx_log_t* log) {
net_instaweb::log_message_handler::Install(log);
ParentOrChildInit(log);
// Let SystemCaches know about the various paths we have in configuration
// first, as well as the memcached instances.
for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
e = uninitialized_server_contexts_.end(); p != e; ++p) {
NgxServerContext* server_context = *p;
caches_->RegisterConfig(server_context->config());
}
caches_->RootInit();
}
void NgxRewriteDriverFactory::ChildInit(ngx_log_t* log) {
is_root_process_ = false;
ParentOrChildInit(log);
if (shared_mem_statistics_.get() != NULL) {
shared_mem_statistics_->Init(false, message_handler());
}
caches_->ChildInit();
for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
e = uninitialized_server_contexts_.end(); p != e; ++p) {
NgxServerContext* server_context = *p;
server_context->ChildInit();
}
uninitialized_server_contexts_.clear();
}
// Initializes global statistics object if needed, using factory to
// help with the settings if needed.
// Note: does not call set_statistics() on the factory.
Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
const NgxRewriteOptions& options) {
if (shared_mem_statistics_.get() == NULL) {
shared_mem_statistics_.reset(AllocateAndInitSharedMemStatistics(
"global", options));
}
DCHECK(!statistics_frozen_);
statistics_frozen_ = true;
SetStatistics(shared_mem_statistics_.get());
return shared_mem_statistics_.get();
}
SharedMemStatistics* NgxRewriteDriverFactory::
AllocateAndInitSharedMemStatistics(
const StringPiece& name,
const NgxRewriteOptions& options) {
GoogleString log_filename;
bool logging_enabled = false;
if (!options.log_dir().empty()) {
// Only enable statistics logging if a log_dir() is actually specified.
log_filename = StrCat(options.log_dir(), "/stats_log_", name);
logging_enabled = options.statistics_logging_enabled();
}
// Note that we create the statistics object in the parent process, and
// it stays around in the kids but gets reinitialized for them
// inside ChildInit(), called from pagespeed_child_init.
SharedMemStatistics* stats = new SharedMemStatistics(
options.statistics_logging_interval_ms(),
options.statistics_logging_max_file_size_kb(),
log_filename, logging_enabled,
StrCat(filename_prefix(), name), shared_mem_runtime(), message_handler(),
file_system(), timer());
InitStats(stats);
stats->Init(true, message_handler());
return stats;
void NgxRewriteDriverFactory::SetCircularBuffer(
SharedCircularBuffer* buffer) {
ngx_shared_circular_buffer_ = buffer;
}
void NgxRewriteDriverFactory::SetServerContextMessageHandler(
@@ -417,7 +266,9 @@ void NgxRewriteDriverFactory::SetServerContextMessageHandler(
NgxMessageHandler* handler = new NgxMessageHandler(
thread_system()->NewMutex());
handler->set_log(log);
handler->set_buffer(shared_circular_buffer_.get());
// The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
// been raised from its default of 0.
handler->set_buffer(ngx_shared_circular_buffer_);
server_context_message_handlers_.insert(handler);
defer_cleanup(new Deleter<NgxMessageHandler>(handler));
server_context->set_message_handler(handler);
@@ -432,8 +283,6 @@ void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
// Init Ngx-specific stats.
NgxServerContext::InitStats(statistics);
InPlaceResourceRecorder::InitStats(statistics);
statistics->AddVariable(kShutdownCount);
}
} // namespace net_instaweb
src/ngx_rewrite_driver_factory.h
+18 -63
View File
@@ -39,7 +39,6 @@ extern "C" {
namespace net_instaweb {
class AbstractSharedMem;
class NgxMessageHandler;
class NgxRewriteOptions;
class NgxServerContext;
@@ -47,18 +46,18 @@ class NgxThreadSystem;
class NgxUrlAsyncFetcher;
class SharedCircularBuffer;
class SharedMemRefererStatistics;
class SharedMemStatistics;
class SlowWorker;
class StaticAssetManager;
class Statistics;
class SystemCaches;
class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
public:
static const char kStaticAssetPrefix[];
// We take ownership of the thread system.
explicit NgxRewriteDriverFactory(NgxThreadSystem* ngx_thread_system);
explicit NgxRewriteDriverFactory(NgxThreadSystem* ngx_thread_system,
StringPiece hostname,
int port);
virtual ~NgxRewriteDriverFactory();
virtual Hasher* NewHasher();
virtual UrlAsyncFetcher* DefaultAsyncUrlFetcher();
@@ -67,73 +66,36 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
virtual FileSystem* DefaultFileSystem();
virtual Timer* DefaultTimer();
virtual NamedLockManager* DefaultLockManager();
virtual void SetupCaches(ServerContext* server_context);
// Create a new RewriteOptions. In this implementation it will be an
// NgxRewriteOptions.
virtual RewriteOptions* NewRewriteOptions();
// Initializes the StaticAssetManager.
virtual void InitStaticAssetManager(
StaticAssetManager* static_asset_manager);
// Print out details of all the connections to memcached servers.
void PrintMemCacheStats(GoogleString* out);
bool InitNgxUrlAsyncFetcher();
// Check resolver configured or not.
bool CheckResolver();
// Release all the resources. It also calls the base class ShutDown to
// release the base class resources.
// Initializes all the statistics objects created transitively by
// NgxRewriteDriverFactory, including nginx-specific and
// platform-independent statistics.
static void InitStats(Statistics* statistics);
virtual void ShutDown();
virtual void StopCacheActivity();
NgxServerContext* MakeNgxServerContext();
ServerContext* NewServerContext();
AbstractSharedMem* shared_mem_runtime() const {
return shared_mem_runtime_.get();
}
SystemCaches* caches() { return caches_.get(); }
// Starts pagespeed threads if they've not been started already. Must be
// called after the caller has finished any forking it intends to do.
void StartThreads();
// This helper method contains init procedures invoked by both RootInit()
// and ChildInit()
void ParentOrChildInit(ngx_log_t* log);
// For shared memory resources the general setup we follow is to have the
// first running process (aka the root) create the necessary segments and
// fill in their shared data structures, while processes created to actually
// handle requests attach to already existing shared data structures.
//
// During normal server startup[1], RootInit() is called from the nginx hooks
// in the root process for the first task, and then ChildInit() is called in
// any child process.
//
// Keep in mind, however, that when fork() is involved a process may
// effectively see both calls, in which case the 'ChildInit' call would
// come second and override the previous root status. Both calls are also
// invoked in the debug single-process mode.
//
// [1] Besides normal startup, nginx also uses a temporary process to
// syntax check the config file. That basically looks like a complete
// normal startup and shutdown to the code.
bool is_root_process() const { return is_root_process_; }
void RootInit(ngx_log_t* log);
void ChildInit(ngx_log_t* log);
void SharedCircularBufferInit(bool is_root);
// Build global shared-memory statistics. This is invoked if at least
// one server context (global or VirtualHost) enables statistics.
Statistics* MakeGlobalSharedMemStatistics(const NgxRewriteOptions& options);
// Creates and ::Initializes a shared memory statistics object.
SharedMemStatistics* AllocateAndInitSharedMemStatistics(
const StringPiece& name, const NgxRewriteOptions& options);
void SetServerContextMessageHandler(ServerContext* server_context,
ngx_log_t* log);
NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
virtual void NonStaticInitStats(Statistics* statistics) {
InitStats(statistics);
}
void set_main_conf(NgxRewriteOptions* main_conf) { main_conf_ = main_conf; }
bool use_per_vhost_statistics() const {
@@ -148,12 +110,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
void set_install_crash_handler(bool x) {
install_crash_handler_ = x;
}
bool message_buffer_size() const {
return message_buffer_size_;
}
void set_message_buffer_size(int x) {
message_buffer_size_ = x;
}
void set_resolver(ngx_resolver_t* resolver) {
resolver_ = resolver;
}
@@ -179,33 +135,28 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
return true;
}
void LoggingInit(ngx_log_t* log);
virtual void ShutDownMessageHandlers();
virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
private:
NgxThreadSystem* ngx_thread_system_;
Timer* timer_;
scoped_ptr<AbstractSharedMem> shared_mem_runtime_;
// main_conf will have only options set in the main block. It may be NULL,
// and we do not take ownership.
NgxRewriteOptions* main_conf_;
typedef std::set<NgxServerContext*> NgxServerContextSet;
NgxServerContextSet uninitialized_server_contexts_;
// Manages all our caches & lock managers.
scoped_ptr<SystemCaches> caches_;
bool threads_started_;
// If true, we'll have a separate statistics object for each vhost
// (along with a global aggregate), rather than just a single object
// aggregating all of them.
bool use_per_vhost_statistics_;
bool is_root_process_;
NgxMessageHandler* ngx_message_handler_;
NgxMessageHandler* ngx_html_parse_message_handler_;
bool install_crash_handler_;
int message_buffer_size_;
scoped_ptr<SharedCircularBuffer> shared_circular_buffer_;
scoped_ptr<SharedMemStatistics> shared_mem_statistics_;
bool statistics_frozen_;
NgxUrlAsyncFetcher* ngx_url_async_fetcher_;
ngx_log_t* log_;
@@ -216,6 +167,10 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
NgxMessageHandlerSet server_context_message_handlers_;
// Owned by the superclass.
// TODO(jefftk): merge the nginx and apache ways of doing this.
SharedCircularBuffer* ngx_shared_circular_buffer_;
DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
};
src/ngx_server_context.cc
+1 -1
View File
@@ -84,7 +84,7 @@ void NgxServerContext::CreateLocalStatistics(
Statistics* global_statistics) {
local_statistics_ =
ngx_factory_->AllocateAndInitSharedMemStatistics(
hostname_identifier(), *config());
true /* local */, hostname_identifier(), *config());
split_statistics_.reset(new SplitStatistics(
ngx_factory_->thread_system(), local_statistics_, global_statistics));
// local_statistics_ was ::InitStat'd by AllocateAndInitSharedMemStatistics,
src/pthread_shared_mem.cc
-257
View File
@@ -1,257 +0,0 @@
// Copyright 2011 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: morlovich@google.com (Maksim Orlovich)
#include "pthread_shared_mem.h"
#include <fcntl.h>
#include <pthread.h>
#include <sys/mman.h>
#include <unistd.h>
#include <cerrno>
#include <cstddef>
#include <map>
#include <utility>
#include "net/instaweb/util/public/abstract_shared_mem.h"
#include "net/instaweb/util/public/abstract_mutex.h"
#include "net/instaweb/util/public/basictypes.h"
#include "net/instaweb/util/public/message_handler.h"
#include "net/instaweb/util/public/string.h"
namespace net_instaweb {
namespace ngx {
namespace {
// This implementation relies on readonly copies of old memory and shared R/W
// mappings being kept across a fork. It simply stashes addresses of
// shared mmap segments into a map where kid processes can pick them up.
// close() a fd logging failure and dealing with EINTR.
void CheckedClose(int fd, MessageHandler* message_handler) {
while (close(fd) != 0) {
if (errno != EINTR) {
message_handler->Message(kWarning, "Problem closing SHM segment fd:%d",
errno);
return;
}
}
}
// Unlike PthreadMutex this doesn't own the lock, but rather refers to an
// external one.
class PthreadSharedMemMutex : public AbstractMutex {
public:
explicit PthreadSharedMemMutex(pthread_mutex_t* external_mutex)
: external_mutex_(external_mutex) {}
virtual bool TryLock() {
return (pthread_mutex_trylock(external_mutex_) == 0);
}
virtual void Lock() {
pthread_mutex_lock(external_mutex_);
}
virtual void Unlock() {
pthread_mutex_unlock(external_mutex_);
}
private:
pthread_mutex_t* external_mutex_;
DISALLOW_COPY_AND_ASSIGN(PthreadSharedMemMutex);
};
class PthreadSharedMemSegment : public AbstractSharedMemSegment {
public:
// We will be representing memory mapped in the [base, base + size) range.
PthreadSharedMemSegment(char* base, size_t size, MessageHandler* handler)
: base_(base),
size_(size) {
}
virtual ~PthreadSharedMemSegment() {
}
virtual volatile char* Base() {
return base_;
}
virtual size_t SharedMutexSize() const {
return sizeof(pthread_mutex_t);
}
virtual bool InitializeSharedMutex(size_t offset, MessageHandler* handler) {
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0) {
handler->Message(kError, "pthread_mutexattr_init failed with errno:%d",
errno);
return false;
}
if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) != 0) {
pthread_mutexattr_destroy(&attr);
handler->Message(
kError, "pthread_mutexattr_setpshared failed with errno:%d", errno);
return false;
}
if (pthread_mutex_init(MutexPtr(offset), &attr) != 0) {
pthread_mutexattr_destroy(&attr);
handler->Message(kError, "pthread_mutex_init failed with errno:%d",
errno);
return false;
}
pthread_mutexattr_destroy(&attr);
return true;
}
virtual AbstractMutex* AttachToSharedMutex(size_t offset) {
return new PthreadSharedMemMutex(MutexPtr(offset));
}
private:
pthread_mutex_t* MutexPtr(size_t offset) {
return reinterpret_cast<pthread_mutex_t*>(base_ + offset);
}
char* const base_;
const size_t size_;
DISALLOW_COPY_AND_ASSIGN(PthreadSharedMemSegment);
};
pthread_mutex_t segment_bases_lock = PTHREAD_MUTEX_INITIALIZER;
} // namespace
size_t PthreadSharedMem::s_instance_count_ = 0;
PthreadSharedMem::SegmentBaseMap* PthreadSharedMem::segment_bases_ = NULL;
PthreadSharedMem::PthreadSharedMem() {
instance_number_ = ++s_instance_count_;
}
PthreadSharedMem::~PthreadSharedMem() {
}
size_t PthreadSharedMem::SharedMutexSize() const {
return sizeof(pthread_mutex_t);
}
AbstractSharedMemSegment* PthreadSharedMem::CreateSegment(
const GoogleString& name, size_t size, MessageHandler* handler) {
GoogleString prefixed_name = PrefixSegmentName(name);
// Create the memory
int fd = open("/dev/zero", O_RDWR);
if (fd == -1) {
handler->Message(kError, "Unable to create SHM segment %s, errno=%d.",
prefixed_name.c_str(), errno);
return NULL;
}
// map it
char* base = reinterpret_cast<char*>(
mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
CheckedClose(fd, handler);
if (base == MAP_FAILED) {
return NULL;
}
SegmentBaseMap* bases = AcquireSegmentBases();
(*bases)[prefixed_name] = base;
UnlockSegmentBases();
return new PthreadSharedMemSegment(base, size, handler);
}
AbstractSharedMemSegment* PthreadSharedMem::AttachToSegment(
const GoogleString& name, size_t size, MessageHandler* handler) {
GoogleString prefixed_name = PrefixSegmentName(name);
SegmentBaseMap* bases = AcquireSegmentBases();
SegmentBaseMap::const_iterator i = bases->find(prefixed_name);
if (i == bases->end()) {
handler->Message(kError, "Unable to find SHM segment %s to attach to.",
prefixed_name.c_str());
UnlockSegmentBases();
return NULL;
}
char* base = i->second;
UnlockSegmentBases();
return new PthreadSharedMemSegment(base, size, handler);
}
void PthreadSharedMem::DestroySegment(const GoogleString& name,
MessageHandler* handler) {
GoogleString prefixed_name = PrefixSegmentName(name);
// Note that in the process state children will not see any mutations
// we make here, so it acts mostly for checking in that case.
SegmentBaseMap* bases = AcquireSegmentBases();
SegmentBaseMap::iterator i = bases->find(prefixed_name);
if (i != bases->end()) {
bases->erase(i);
if (bases->empty()) {
delete segment_bases_;
segment_bases_ = NULL;
}
} else {
handler->Message(kError, "Attempt to destroy unknown SHM segment %s.",
prefixed_name.c_str());
}
UnlockSegmentBases();
}
PthreadSharedMem::SegmentBaseMap* PthreadSharedMem::AcquireSegmentBases() {
PthreadSharedMemMutex lock(&segment_bases_lock);
lock.Lock();
if (segment_bases_ == NULL) {
segment_bases_ = new SegmentBaseMap();
}
return segment_bases_;
}
void PthreadSharedMem::UnlockSegmentBases() {
PthreadSharedMemMutex lock(&segment_bases_lock);
lock.Unlock();
}
GoogleString PthreadSharedMem::PrefixSegmentName(const GoogleString& name) {
GoogleString res;
StrAppend(&res, "[", IntegerToString(instance_number_), "]", name);
return res;
}
void PthreadSharedMem::Terminate() {
// Clean up the local memory associated with the maps to shared memory
// storage.
PthreadSharedMemMutex lock(&segment_bases_lock);
lock.Lock();
if (segment_bases_ != NULL) {
delete segment_bases_;
segment_bases_ = NULL;
}
lock.Unlock();
}
} // namespace ngx
} // namespace net_instaweb
src/pthread_shared_mem.h
-88
View File
@@ -1,88 +0,0 @@
// Copyright 2011 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: morlovich@google.com (Maksim Orlovich)
#ifndef NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_
#define NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_
#include <cstddef>
#include <map>
#include "net/instaweb/util/public/abstract_shared_mem.h"
#include "net/instaweb/util/public/basictypes.h"
#include "net/instaweb/util/public/string.h"
namespace net_instaweb {
class MessageHandler;
namespace ngx {
// POSIX shared memory support, using mmap/pthread_mutexattr_setpshared
// Supports both processes and threads, but processes that want to access it
// must be results of just fork (without exec), and all the CreateSegment
// calls must occur before the fork.
//
// This implementation is also not capable of deallocating segments except
// at exit, so it should not be used when the set of segments may be dynamic.
class PthreadSharedMem : public AbstractSharedMem {
public:
PthreadSharedMem();
virtual ~PthreadSharedMem();
virtual size_t SharedMutexSize() const;
virtual AbstractSharedMemSegment* CreateSegment(
const GoogleString& name, size_t size, MessageHandler* handler);
virtual AbstractSharedMemSegment* AttachToSegment(
const GoogleString& name, size_t size, MessageHandler* handler);
virtual void DestroySegment(const GoogleString& name,
MessageHandler* handler);
// Frees all lazy-initialized memory used to track shared-memory segments.
static void Terminate();
private:
typedef std::map<GoogleString, char*> SegmentBaseMap;
// Accessor for below. Note that the segment_bases_lock will be held at exit.
static SegmentBaseMap* AcquireSegmentBases();
static void UnlockSegmentBases();
// Prefixes the passed in segment name with the current instance number.
GoogleString PrefixSegmentName(const GoogleString& name);
// The root process stores segment locations here. Child processes will
// inherit a readonly copy of this map after the fork. Note that this is
// initialized in a thread-unsafe manner, given the above assumptions.
static SegmentBaseMap* segment_bases_;
// Holds the number of times a PthreadSharedMem has been created.
static size_t s_instance_count_;
// Used to prefix segment names, so that when two runtimes are active at the
// same moment they will not have overlapping segment names. This occurs in
// ngx_pagespeed during a configuration reload, where first a new factory is
// created, before destroying the old one.
size_t instance_number_;
DISALLOW_COPY_AND_ASSIGN(PthreadSharedMem);
};
} // namespace ngx
} // namespace net_instaweb
#endif // NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_
test/nginx_system_test.sh
+17 -7
View File
@@ -583,6 +583,12 @@ sleep .1
OUT=$($FETCH_CMD)
check_not_from "$OUT" fgrep "<style>"
# If DisableRewriteOnNoTransform is turned off, verify that the rewriting
# applies even if Cache-control: no-transform is set.
start_test rewrite on Cache-control: no-transform
URL=$TEST_ROOT/disable_no_transform/index.html?PageSpeedFilters=inline_css
fetch_until -save -recursive $URL 'grep -c style' 2
WGET_ARGS=""
start_test PageSpeedShardDomain directive in location block
fetch_until -save $TEST_ROOT/shard/shard.html 'grep -c \.pagespeed\.' 4
@@ -1589,9 +1595,9 @@ OPTIONS_HASH=$(grep "^pagespeed\.criticalImagesBeaconInit" \
# Send a beacon response using POST indicating that Puzzle.jpg is a critical
# image.
BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon"
BEACON_DATA="url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
BEACON_DATA+="image_rewriting%2Frewrite_images.html"
BEACON_DATA+="&oh=$OPTIONS_HASH&ci=2932493096"
BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F"
BEACON_URL+="image_rewriting%2Frewrite_images.html"
BEACON_DATA="oh=$OPTIONS_HASH&ci=2932493096"
# See the comments about 204 responses and --no-http-keep-alive above.
OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
wget -q --save-headers -O - --no-http-keep-alive \
@@ -1605,13 +1611,17 @@ http_proxy=$SECONDARY_HOSTNAME \
# Now test sending a beacon with a GET request, instead of POST. Indicate that
# Puzzle.jpg and Cuppa.png are the critical images. In practice we expect only
# POSTs to be used by the critical image beacon, but both code paths are
# supported.
# supported. We need to do this several times since 80% support is required
# for an image to be considered critical.
# Add the hash for Cuppa.png to BEACON_DATA, which will be used as the query
# params for the GET.
BEACON_DATA+=",2644480723"
OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
$WGET_DUMP "$BEACON_URL?$BEACON_DATA")
check_from "$OUT" egrep -q "HTTP/1[.]. 204"
for i in {1..4}; do
OUT=$(env http_proxy=$SECONDARY_HOSTNAME \
$WGET_DUMP "$BEACON_URL&$BEACON_DATA")
check_from "$OUT" egrep -q "HTTP/1[.]. 204"
done
# Now only BikeCrashIcn.png should be lazyloaded.
http_proxy=$SECONDARY_HOSTNAME \
fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 1
test/pagespeed_test.conf.template
+9
View File
@@ -523,6 +523,15 @@ http {
pagespeed RewriteLevel PassThrough;
pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images,convert_jpeg_to_webp,defer_javascript;
}
location /mod_pagespeed_test/disable_no_transform/index.html {
pagespeed DisableRewriteOnNoTransform off;
}
location /mod_pagespeed_test/disable_no_transform/disable_no_transform.css {
add_header 'Cache-Control' 'no-transform';
}
# uncomment the following two lines if you're testing memcached
#pagespeed MemcachedServers "localhost:11211";
#pagespeed MemcachedThreads 1;