diff --git a/README.md b/README.md index d33bef961..73d36146e 100644 --- a/README.md +++ b/README.md @@ -37,21 +37,21 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p ```bash $ cd ~ - $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.1-beta.zip - $ unzip v1.7.30.1-beta.zip # or unzip v1.7.30.1-beta - $ cd ngx_pagespeed-1.7.30.1-beta/ - $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.1.tar.gz - $ tar -xzvf 1.7.30.1.tar.gz # expands to psol/ + $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.2-beta.zip + $ unzip v1.7.30.2-beta.zip # or unzip v1.7.30.2-beta + $ cd ngx_pagespeed-1.7.30.2-beta/ + $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.2.tar.gz + $ tar -xzvf 1.7.30.2.tar.gz # expands to psol/ ``` 3. Download and build nginx: ```bash $ # check http://nginx.org/en/download.html for the latest version - $ wget http://nginx.org/download/nginx-1.4.3.tar.gz - $ tar -xvzf nginx-1.4.3.tar.gz - $ cd nginx-1.4.3/ - $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.1-beta + $ wget http://nginx.org/download/nginx-1.4.4.tar.gz + $ tar -xvzf nginx-1.4.4.tar.gz + $ cd nginx-1.4.4/ + $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.2-beta $ make $ sudo make install ``` @@ -94,7 +94,7 @@ To confirm that the module is loaded, fetch a page and check that you see the ```bash $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed -X-Page-Speed: 1.7.30.1-... +X-Page-Speed: 1.7.30.2-... ``` Looking at the source of a few pages you should see various changes, such as diff --git a/config b/config index 1309facf1..4cdd8ee05 100644 --- a/config +++ b/config @@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then echo " You need to separately download the pagespeed library:" echo "" echo " $ cd /path/to/ngx_pagespeed" - echo " $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.1.tar.gz" - echo " $ tar -xzvf 1.7.30.1.tar.gz # expands to psol/" + echo " $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.2.tar.gz" + echo " $ tar -xzvf 1.7.30.2.tar.gz # expands to psol/" echo "" echo " Or see the installation instructions:" echo " https://github.com/pagespeed/ngx_pagespeed#how-to-build" @@ -194,7 +194,8 @@ if [ $ngx_found = yes ]; then echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES else cat << END -$0: error: module ngx_pagespeed requires the pagespeed optimization library +$0: error: module ngx_pagespeed requires the pagespeed optimization library. +Look in obj/autoconf.err for more details. END exit 1 fi diff --git a/src/ngx_fetch.cc b/src/ngx_fetch.cc index 995ecc879..e18a6502c 100644 --- a/src/ngx_fetch.cc +++ b/src/ngx_fetch.cc @@ -142,19 +142,26 @@ namespace net_instaweb { // The host is either a domain name or an IP address. First check // if it's a valid IP address and only if that fails fall back to // using the DNS resolver. - GoogleString s_ipaddress(reinterpret_cast(url_.host.data), - url_.host.len); + + // Maybe we have a Proxy. + ngx_url_t* tmp_url = &url_; + if (0 != fetcher_->proxy_.url.len) { + tmp_url = &fetcher_->proxy_; + } + + GoogleString s_ipaddress(reinterpret_cast(tmp_url->host.data), + tmp_url->host.len); ngx_memzero(&sin_, sizeof(sin_)); sin_.sin_family = AF_INET; - sin_.sin_port = htons(url_.port); + sin_.sin_port = htons(tmp_url->port); sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str()); if (sin_.sin_addr.s_addr == INADDR_NONE) { // inet_addr returned INADDR_NONE, which means the hostname // isn't a valid IP address. Check DNS. ngx_resolver_ctx_t temp; - temp.name.data = url_.host.data; - temp.name.len = url_.host.len; + temp.name.data = tmp_url->host.data; + temp.name.len = tmp_url->host.len; resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp); if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) { // TODO(oschaaf): this spams the log, but is useful in the fetcher's @@ -166,8 +173,8 @@ namespace net_instaweb { } resolver_ctx_->data = this; - resolver_ctx_->name.data = url_.host.data; - resolver_ctx_->name.len = url_.host.len; + resolver_ctx_->name.data = tmp_url->host.data; + resolver_ctx_->name.len = tmp_url->host.len; #if (nginx_version < 1005008) resolver_ctx_->type = NGX_RESOLVE_A; @@ -263,37 +270,14 @@ namespace net_instaweb { return false; } str_url_.copy(reinterpret_cast(url_.url.data), str_url_.length(), 0); - size_t scheme_offset; - u_short port; - if (ngx_strncasecmp(url_.url.data, reinterpret_cast( - const_cast("http://")), 7) == 0) { - scheme_offset = 7; - port = 80; - } else if (ngx_strncasecmp(url_.url.data, reinterpret_cast( - const_cast("https://")), 8) == 0) { - scheme_offset = 8; - port = 443; - } else { - scheme_offset = 0; - port = 80; - } - url_.url.data += scheme_offset; - url_.url.len -= scheme_offset; - url_.default_port = port; - // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875 - url_.no_resolve = 0; - url_.uri_part = 1; - - if (ngx_parse_url(pool_, &url_) == NGX_OK) { - return true; - } - return false; + return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_); } // Issue a request after the resolver is done void NgxFetch::NgxFetchResolveDone(ngx_resolver_ctx_t* resolver_ctx) { NgxFetch* fetch = static_cast(resolver_ctx->data); + NgxUrlAsyncFetcher* fetcher = fetch->fetcher_; if (resolver_ctx->state != NGX_OK) { if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) { ngx_del_timer(fetch->timeout_event()); @@ -322,6 +306,11 @@ namespace net_instaweb { fetch->sin_.sin_family = AF_INET; fetch->sin_.sin_port = htons(fetch->url_.port); + // Maybe we have Proxy + if (0 != fetcher->proxy_.url.len) { + fetch->sin_.sin_port = htons(fetcher->proxy_.port); + } + char* ip_address = inet_ntoa(fetch->sin_.sin_addr); fetch->message_handler()->Message( diff --git a/src/ngx_url_async_fetcher.cc b/src/ngx_url_async_fetcher.cc index 9db9fc8d0..ac41bd417 100644 --- a/src/ngx_url_async_fetcher.cc +++ b/src/ngx_url_async_fetcher.cc @@ -66,10 +66,10 @@ namespace net_instaweb { mutex_(NULL) { resolver_timeout_ = resolver_timeout; fetch_timeout_ = fetch_timeout; - ngx_memzero(&url_, sizeof(url_)); + ngx_memzero(&proxy_, sizeof(proxy_)); if (proxy != NULL && *proxy != '\0') { - url_.url.data = reinterpret_cast(const_cast(proxy)); - url_.url.len = ngx_strlen(proxy); + proxy_.url.data = reinterpret_cast(const_cast(proxy)); + proxy_.url.len = ngx_strlen(proxy); } mutex_ = thread_system_->NewMutex(); log_ = log; @@ -106,6 +106,36 @@ namespace net_instaweb { } } + + bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) { + size_t scheme_offset; + u_short port; + if (ngx_strncasecmp(url->url.data, reinterpret_cast( + const_cast("http://")), 7) == 0) { + scheme_offset = 7; + port = 80; + } else if (ngx_strncasecmp(url->url.data, reinterpret_cast( + const_cast("https://")), 8) == 0) { + scheme_offset = 8; + port = 443; + } else { + scheme_offset = 0; + port = 80; + } + + url->url.data += scheme_offset; + url->url.len -= scheme_offset; + url->default_port = port; + // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875 + url->no_resolve = 0; + url->uri_part = 1; + + if (ngx_parse_url(pool, url) == NGX_OK) { + return true; + } + return false; + } + // If there are still active requests, cancel them. void NgxUrlAsyncFetcher::CancelActiveFetches() { // TODO(oschaaf): this seems tricky, this may end up calling @@ -167,15 +197,15 @@ namespace net_instaweb { command_connection_->read->handler = CommandHandler; ngx_add_event(command_connection_->read, NGX_READ_EVENT, 0); - if (url_.url.len == 0) { + if (proxy_.url.len == 0) { return true; } // TODO(oschaaf): shouldn't we do this earlier? Do we need to clean // up when parsing the url failed? - if (ngx_parse_url(pool_, &url_) != NGX_OK) { + if (!ParseUrl(&proxy_, pool_)) { ngx_log_error(NGX_LOG_ERR, log_, 0, - "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &url_.url); + "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url); return false; } return true; diff --git a/src/ngx_url_async_fetcher.h b/src/ngx_url_async_fetcher.h index d2b717f02..8cd96021d 100644 --- a/src/ngx_url_async_fetcher.h +++ b/src/ngx_url_async_fetcher.h @@ -115,13 +115,14 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher { private: static void TimeoutHandler(ngx_event_t* tev); + static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool); friend class NgxFetch; NgxFetchPool active_fetches_; // Add the pending task to this list NgxFetchPool pending_fetches_; NgxFetchPool completed_fetches_; - ngx_url_t url_; + ngx_url_t proxy_; int fetchers_count_; bool shutdown_;