From ba1c1853a79a5930ca7db7a6b368859f805df98d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 29 Aug 2019 15:59:16 +0200 Subject: swh: Add hooks for rate limiting handling. * guix/swh.scm (%allow-request?, %save-rate-limit-reset-time) (%general-rate-limit-reset-time): New variables. (request-rate-limit-reached?, update-rate-limit-reset-time!): New procedures. (call): Call '%allow-request?'. Change 'swh-error' protocol to pass METHOD in addition to URL. * tests/swh.scm ("rate limit reached") ("%allow-request? and request-rate-limit-reached?"): New tests. --- guix/swh.scm | 84 +++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 20 deletions(-) (limited to 'guix/swh.scm') diff --git a/guix/swh.scm b/guix/swh.scm index c253e217da..42f38ee048 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -20,6 +20,7 @@ #:use-module (guix base16) #:use-module (guix build utils) #:use-module ((guix build syscalls) #:select (mkdtemp!)) + #:use-module (web uri) #:use-module (web client) #:use-module (web response) #:use-module (json) @@ -32,6 +33,9 @@ #:use-module (ice-9 popen) #:use-module ((ice-9 ftw) #:select (scandir)) #:export (%swh-base-url + %allow-request? + + request-rate-limit-reached? origin? origin-id @@ -196,31 +200,71 @@ Software Heritage." ((? string? str) str) ((? null?) #f))) +(define %allow-request? + ;; Takes a URL and method (e.g., the 'http-get' procedure) and returns true + ;; to keep going. This can be used to disallow a requests when + ;; 'request-rate-limit-reached?' returns true, for instance. + (make-parameter (const #t))) + +;; The time when the rate limit for "/origin/save" POST requests and that of +;; other requests will be reset. +;; See . +(define %save-rate-limit-reset-time 0) +(define %general-rate-limit-reset-time 0) + +(define (request-rate-limit-reached? url method) + "Return true if the rate limit has been reached for URI." + (define uri + (string->uri url)) + + (define reset-time + (if (and (eq? method http-post) + (string-prefix? "/api/1/origin/save/" (uri-path uri))) + %save-rate-limit-reset-time + %general-rate-limit-reset-time)) + + (< (car (gettimeofday)) reset-time)) + +(define (update-rate-limit-reset-time! url method response) + "Update the rate limit reset time for URL and METHOD based on the headers in +RESPONSE." + (let ((uri (string->uri url))) + (match (assq-ref (response-headers response) 'x-ratelimit-reset) + ((= string->number (? number? reset)) + (if (and (eq? method http-post) + (string-prefix? "/api/1/origin/save/" (uri-path uri))) + (set! %save-rate-limit-reset-time reset) + (set! %general-rate-limit-reset-time reset))) + (_ + #f)))) + (define* (call url decode #:optional (method http-get) #:key (false-if-404? #t)) "Invoke the endpoint at URL using METHOD. Decode the resulting JSON body using DECODE, a one-argument procedure that takes an input port. When FALSE-IF-404? is true, return #f upon 404 responses." - (let*-values (((response port) - (method url #:streaming? #t))) - ;; See . - (match (assq-ref (response-headers response) 'x-ratelimit-remaining) - (#f #t) - ((? (compose zero? string->number)) - (throw 'swh-error url response)) - (_ #t)) - - (cond ((= 200 (response-code response)) - (let ((result (decode port))) - (close-port port) - result)) - ((and false-if-404? - (= 404 (response-code response))) - (close-port port) - #f) - (else - (close-port port) - (throw 'swh-error url response))))) + (and ((%allow-request?) url method) + (let*-values (((response port) + (method url #:streaming? #t))) + ;; See . + (match (assq-ref (response-headers response) 'x-ratelimit-remaining) + (#f #t) + ((? (compose zero? string->number)) + (update-rate-limit-reset-time! url method response) + (throw 'swh-error url method response)) + (_ #t)) + + (cond ((= 200 (response-code response)) + (let ((result (decode port))) + (close-port port) + result)) + ((and false-if-404? + (= 404 (response-code response))) + (close-port port) + #f) + (else + (close-port port) + (throw 'swh-error url method response)))))) (define-syntax define-query (syntax-rules (path) -- cgit v1.2.3 From d370cc73193f42fb86f08237b4ebb612ef822ae7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 29 Aug 2019 16:02:29 +0200 Subject: swh: Make 'commit-id?' public. * guix/swh.scm (commit-id?): Make public. --- guix/swh.scm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'guix/swh.scm') diff --git a/guix/swh.scm b/guix/swh.scm index 42f38ee048..01648a1ebe 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -105,6 +105,8 @@ request-cooking vault-fetch + commit-id? + swh-download)) ;;; Commentary: @@ -568,7 +570,7 @@ requested bundle cooking, waiting for completion...~%")) (define (commit-id? reference) "Return true if REFERENCE is likely a commit ID, false otherwise---e.g., if -it is a tag name." +it is a tag name. This is based on a simple heuristic so use with care!" (and (= (string-length reference) 40) (string-every char-set:hex-digit reference))) -- cgit v1.2.3 From 76073d29e11c71d3678efd44db646852b5502e55 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sun, 1 Sep 2019 13:24:03 +0200 Subject: Add (guix json). * guix/swh.scm (define-json-reader, define-json-mapping): Move to... * guix/json.scm: ... here. New file. * Makefile.am (MODULES): Add it. --- Makefile.am | 1 + guix/json.scm | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ guix/swh.scm | 35 +-------------------------------- 3 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 guix/json.scm (limited to 'guix/swh.scm') diff --git a/Makefile.am b/Makefile.am index fa6bf8fe80..7b96c9473c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -68,6 +68,7 @@ MODULES = \ guix/cpio.scm \ guix/deprecation.scm \ guix/docker.scm \ + guix/json.scm \ guix/records.scm \ guix/pki.scm \ guix/progress.scm \ diff --git a/guix/json.scm b/guix/json.scm new file mode 100644 index 0000000000..20f0bd8f13 --- /dev/null +++ b/guix/json.scm @@ -0,0 +1,62 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018, 2019 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix json) + #:use-module (json) + #:use-module (srfi srfi-9) + #:export (define-json-mapping)) + +;;; Commentary: +;;; +;;; Helpers to map JSON objects to SRFI-9 records. Taken from (guix swh). +;;; +;;; Code: + +(define-syntax-rule (define-json-reader json->record ctor spec ...) + "Define JSON->RECORD as a procedure that converts a JSON representation, +read from a port, string, or hash table, into a record created by CTOR and +following SPEC, a series of field specifications." + (define (json->record input) + (let ((table (cond ((port? input) + (json->scm input)) + ((string? input) + (json-string->scm input)) + ((or (null? input) (pair? input)) + input)))) + (let-syntax ((extract-field (syntax-rules () + ((_ table (field key json->value)) + (json->value (assoc-ref table key))) + ((_ table (field key)) + (assoc-ref table key)) + ((_ table (field)) + (assoc-ref table + (symbol->string 'field)))))) + (ctor (extract-field table spec) ...))))) + +(define-syntax-rule (define-json-mapping rtd ctor pred json->record + (field getter spec ...) ...) + "Define RTD as a record type with the given FIELDs and GETTERs, à la SRFI-9, +and define JSON->RECORD as a conversion from JSON to a record of this type." + (begin + (define-record-type rtd + (ctor field ...) + pred + (field getter) ...) + + (define-json-reader json->record ctor + (field spec ...) ...))) diff --git a/guix/swh.scm b/guix/swh.scm index 01648a1ebe..7acad05928 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -21,6 +21,7 @@ #:use-module (guix build utils) #:use-module ((guix build syscalls) #:select (mkdtemp!)) #:use-module (web uri) + #:use-module (guix json) #:use-module (web client) #:use-module (web response) #:use-module (json) @@ -135,40 +136,6 @@ url (string-append url "/"))) -(define-syntax-rule (define-json-reader json->record ctor spec ...) - "Define JSON->RECORD as a procedure that converts a JSON representation, -read from a port, string, or hash table, into a record created by CTOR and -following SPEC, a series of field specifications." - (define (json->record input) - (let ((table (cond ((port? input) - (json->scm input)) - ((string? input) - (json-string->scm input)) - ((or (null? input) (pair? input)) - input)))) - (let-syntax ((extract-field (syntax-rules () - ((_ table (field key json->value)) - (json->value (assoc-ref table key))) - ((_ table (field key)) - (assoc-ref table key)) - ((_ table (field)) - (assoc-ref table - (symbol->string 'field)))))) - (ctor (extract-field table spec) ...))))) - -(define-syntax-rule (define-json-mapping rtd ctor pred json->record - (field getter spec ...) ...) - "Define RTD as a record type with the given FIELDs and GETTERs, à la SRFI-9, -and define JSON->RECORD as a conversion from JSON to a record of this type." - (begin - (define-record-type rtd - (ctor field ...) - pred - (field getter) ...) - - (define-json-reader json->record ctor - (field spec ...) ...))) - (define %date-regexp ;; Match strings like "2014-11-17T22:09:38+01:00" or ;; "2018-09-30T23:20:07.815449+00:00"". -- cgit v1.2.3