[PATCHES] Upgrade sqlite and fix it on systems with page size > 32K

[PATCHES] Upgrade sqlite and fix it on systems with page size > 32K

[Mark H Weaver]

[-- Attachment #1: Type: text/plain, Size: 149 bytes --]

This is needed on systems with 64K page size.  It also upgrades sqlite
to the latest version and computes the source URL automatically.

     Mark



[-- Attachment #2: [PATCH 1/3] gnu: sqlite: Compute source URL from version number --]
[-- Type: text/x-patch, Size: 2430 bytes --]

From 8243f4e481eaf5604dd5c0ad8e069885af40dbdd Mon Sep 17 00:00:00 2001
From: Mark H Weaver <mhw@netris.org>
Date: Wed, 2 Apr 2014 12:02:41 -0400
Subject: [PATCH 1/3] gnu: sqlite: Compute source URL from version number.

* gnu/packages/sqlite.scm (sqlite): Compute the numeric-version from the
  traditional version string, and use it to compute the source URL.
---
 gnu/packages/sqlite.scm | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/gnu/packages/sqlite.scm b/gnu/packages/sqlite.scm
index f688e13..0c8b9d2 100644
--- a/gnu/packages/sqlite.scm
+++ b/gnu/packages/sqlite.scm
@@ -1,5 +1,6 @@
 ;;; GNU Guix --- Functional package management for GNU
 ;;; Copyright © 2013 Cyril Roelandt <tipecaml@gmail.com>
+;;; Copyright © 2014 Mark H Weaver <mhw@netris.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
@@ -21,7 +22,10 @@
   #:use-module (guix packages)
   #:use-module (guix download)
   #:use-module (guix build-system gnu)
-  #:use-module (gnu packages))
+  #:use-module (gnu packages)
+  #:use-module (gnu packages)
+  #:use-module (srfi srfi-26)
+  #:use-module (ice-9 match))
 
 (define-public sqlite
   (package
@@ -32,9 +36,18 @@
             ;; TODO: Download from sqlite.org once this bug :
             ;; http://lists.gnu.org/archive/html/bug-guile/2013-01/msg00027.html
             ;; has been fixed.
-            (uri (string-append
-                  "mirror://sourceforge/sqlite.mirror/SQLite%20"
-                  version "/sqlite-autoconf-3080200.tar.gz"))
+            (uri (let ((numeric-version
+                        (match (string-split version #\.)
+                          ((first-digit other-digits ...)
+                           (string-append first-digit
+                                          (string-pad-right
+                                           (string-concatenate
+                                            (map (cut string-pad <> 2 #\0)
+                                                 other-digits))
+                                           6 #\0))))))
+                   (string-append
+                    "mirror://sourceforge/sqlite.mirror/SQLite%20" version
+                    "/sqlite-autoconf-" numeric-version ".tar.gz")))
             (sha256
              (base32
               "14pg9zlwbwsj5w7f3qr25d3nniyv82gmczwlvpj0i0ic1431v1d0"))))
-- 
1.8.4


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: [PATCH 2/3] gnu: sqlite: Upgrade to 3.8.4.2 --]
[-- Type: text/x-patch, Size: 1153 bytes --]

From 864f748752072eadabb97a390f465f8e052ca7f3 Mon Sep 17 00:00:00 2001
From: Mark H Weaver <mhw@netris.org>
Date: Wed, 2 Apr 2014 12:12:13 -0400
Subject: [PATCH 2/3] gnu: sqlite: Upgrade to 3.8.4.2.

* gnu/packages/sqlite.scm (sqlite): Upgrade to 3.8.4.2.
---
 gnu/packages/sqlite.scm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gnu/packages/sqlite.scm b/gnu/packages/sqlite.scm
index 0c8b9d2..8e976ce 100644
--- a/gnu/packages/sqlite.scm
+++ b/gnu/packages/sqlite.scm
@@ -30,7 +30,7 @@
 (define-public sqlite
   (package
    (name "sqlite")
-   (version "3.8.2")
+   (version "3.8.4.2")
    (source (origin
             (method url-fetch)
             ;; TODO: Download from sqlite.org once this bug :
@@ -50,7 +50,7 @@
                     "/sqlite-autoconf-" numeric-version ".tar.gz")))
             (sha256
              (base32
-              "14pg9zlwbwsj5w7f3qr25d3nniyv82gmczwlvpj0i0ic1431v1d0"))))
+              "19gicv5vdi5c0p8shr1bmihldj409aqz3r4wr7d3pwb6xf1xv4p4"))))
    (build-system gnu-build-system)
    (home-page "http://www.sqlite.org/")
    (synopsis "The SQLite database management system")
-- 
1.8.4


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #4: [PATCH 3/3] gnu: sqlite: Fix sqlite on systems with page size larger than 32K --]
[-- Type: text/x-patch, Size: 7985 bytes --]

From 47b6fcd3d8fbba018f7ddecf86e793de5161489e Mon Sep 17 00:00:00 2001
From: Mark H Weaver <mhw@netris.org>
Date: Wed, 2 Apr 2014 12:14:46 -0400
Subject: [PATCH 3/3] gnu: sqlite: Fix sqlite on systems with page size larger
 than 32K.

Based on a patch by Dan Kennedy <danielk1977@gmail.com>.

* gnu/packages/patches/sqlite-large-page-size-fix.patch: New file.
* gnu/packages/sqlite.scm (sqlite): Add it.
* gnu-system.am (dist_patch_DATA): Add it.
---
 gnu-system.am                                      |   1 +
 .../patches/sqlite-large-page-size-fix.patch       | 180 +++++++++++++++++++++
 gnu/packages/sqlite.scm                            |   4 +-
 3 files changed, 184 insertions(+), 1 deletion(-)
 create mode 100644 gnu/packages/patches/sqlite-large-page-size-fix.patch

diff --git a/gnu-system.am b/gnu-system.am
index 533dbf9..933cdb1 100644
--- a/gnu-system.am
+++ b/gnu-system.am
@@ -324,6 +324,7 @@ dist_patch_DATA =						\
   gnu/packages/patches/slim-config.patch			\
   gnu/packages/patches/slim-sigusr1.patch			\
   gnu/packages/patches/source-highlight-regexrange-test.patch	\
+  gnu/packages/patches/sqlite-large-page-size-fix.patch		\
   gnu/packages/patches/tcsh-fix-autotest.patch			\
   gnu/packages/patches/teckit-cstdio.patch			\
   gnu/packages/patches/valgrind-glibc.patch			\
diff --git a/gnu/packages/patches/sqlite-large-page-size-fix.patch b/gnu/packages/patches/sqlite-large-page-size-fix.patch
new file mode 100644
index 0000000..c561fa2
--- /dev/null
+++ b/gnu/packages/patches/sqlite-large-page-size-fix.patch
@@ -0,0 +1,180 @@
+Add an experimental fix to avoid attempting to mmap memory from an
+offset that is not a multiple of the system page size on systems with
+page sizes larger than 32KB.
+
+Patch by Dan Kennedy <danielk1977@gmail.com>.
+
+--- sqlite-autoconf/sqlite3.c.orig	2014-03-22 23:44:47.055908203 -0400
++++ sqlite-autoconf/sqlite3.c	2014-03-22 23:44:06.716552734 -0400
+@@ -24010,6 +24010,7 @@
+ 
+ /* Forward reference */
+ static int openDirectory(const char*, int*);
++static int unixGetpagesize(void);
+ 
+ /*
+ ** Many system calls are accessed through pointer-to-functions so that
+@@ -24133,6 +24134,9 @@
+ #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent)
+ #endif
+ 
++  { "getpagesize",  (sqlite3_syscall_ptr)unixGetpagesize, 0 },
++#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent)
++
+ }; /* End of the overrideable system calls */
+ 
+ /*
+@@ -27792,6 +27796,36 @@
+   return rc;        
+ }
+ 
++/*
++** Return the system page size.
++**
++** This function should not be called directly by other code in this file. 
++** Instead, it should be called via macro osGetpagesize().
++*/
++static int unixGetpagesize(void){
++#if defined(_BSD_SOURCE)
++  return getpagesize();
++#else
++  return (int)sysconf(_SC_PAGESIZE);
++#endif
++}
++
++/*
++** Return the minimum number of 32KB shm regions that should be mapped at
++** a time, assuming that each mapping must be an integer multiple of the
++** current system page-size.
++**
++** Usually, this is 1. The exception seems to be systems that are configured
++** to use 64KB pages - in this case each mapping must cover at least two
++** shm regions.
++*/
++static int unixShmRegionPerMap(void){
++  int shmsz = 32*1024;            /* SHM region size */
++  int pgsz = osGetpagesize();   /* System page size */
++  assert( ((pgsz-1)&pgsz)==0 );   /* Page size must be a power of 2 */
++  if( pgsz<shmsz ) return 1;
++  return pgsz/shmsz;
++}
+ 
+ /*
+ ** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0.
+@@ -27803,10 +27837,11 @@
+   unixShmNode *p = pFd->pInode->pShmNode;
+   assert( unixMutexHeld() );
+   if( p && p->nRef==0 ){
++    int nShmPerMap = unixShmRegionPerMap();
+     int i;
+     assert( p->pInode==pFd->pInode );
+     sqlite3_mutex_free(p->mutex);
+-    for(i=0; i<p->nRegion; i++){
++    for(i=0; i<p->nRegion; i+=nShmPerMap){
+       if( p->h>=0 ){
+         osMunmap(p->apRegion[i], p->szRegion);
+       }else{
+@@ -28013,6 +28048,8 @@
+   unixShm *p;
+   unixShmNode *pShmNode;
+   int rc = SQLITE_OK;
++  int nShmPerMap = unixShmRegionPerMap();
++  int nReqRegion;
+ 
+   /* If the shared-memory file has not yet been opened, open it now. */
+   if( pDbFd->pShm==0 ){
+@@ -28028,9 +28065,12 @@
+   assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 );
+   assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 );
+ 
+-  if( pShmNode->nRegion<=iRegion ){
++  /* Minimum number of regions required to be mapped. */
++  nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap;
++
++  if( pShmNode->nRegion<nReqRegion ){
+     char **apNew;                      /* New apRegion[] array */
+-    int nByte = (iRegion+1)*szRegion;  /* Minimum required file size */
++    int nByte = nReqRegion*szRegion;   /* Minimum required file size */
+     struct stat sStat;                 /* Used by fstat() */
+ 
+     pShmNode->szRegion = szRegion;
+@@ -28079,17 +28119,19 @@
+ 
+     /* Map the requested memory region into this processes address space. */
+     apNew = (char **)sqlite3_realloc(
+-        pShmNode->apRegion, (iRegion+1)*sizeof(char *)
++        pShmNode->apRegion, nReqRegion*sizeof(char *)
+     );
+     if( !apNew ){
+       rc = SQLITE_IOERR_NOMEM;
+       goto shmpage_out;
+     }
+     pShmNode->apRegion = apNew;
+-    while(pShmNode->nRegion<=iRegion){
++    while( pShmNode->nRegion<nReqRegion ){
++      int nMap = szRegion*nShmPerMap;
++      int i;
+       void *pMem;
+       if( pShmNode->h>=0 ){
+-        pMem = osMmap(0, szRegion,
++        pMem = osMmap(0, nMap,
+             pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, 
+             MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion
+         );
+@@ -28105,8 +28147,11 @@
+         }
+         memset(pMem, 0, szRegion);
+       }
+-      pShmNode->apRegion[pShmNode->nRegion] = pMem;
+-      pShmNode->nRegion++;
++
++      for(i=0; i<nShmPerMap; i++){
++        pShmNode->apRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i];
++      }
++      pShmNode->nRegion += nShmPerMap;
+     }
+   }
+ 
+@@ -28321,19 +28366,6 @@
+ }
+ 
+ /*
+-** Return the system page size.
+-*/
+-static int unixGetPagesize(void){
+-#if HAVE_MREMAP
+-  return 512;
+-#elif defined(_BSD_SOURCE)
+-  return getpagesize();
+-#else
+-  return (int)sysconf(_SC_PAGESIZE);
+-#endif
+-}
+-
+-/*
+ ** Attempt to set the size of the memory mapping maintained by file 
+ ** descriptor pFd to nNew bytes. Any existing mapping is discarded.
+ **
+@@ -28369,8 +28401,12 @@
+   if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;
+ 
+   if( pOrig ){
+-    const int szSyspage = unixGetPagesize();
++#if HAVE_MREMAP
++    i64 nReuse = pFd->mmapSize;
++#else
++    const int szSyspage = osGetpagesize();
+     i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));
++#endif
+     u8 *pReq = &pOrig[nReuse];
+ 
+     /* Unmap any pages of the existing mapping that cannot be reused. */
+@@ -31116,7 +31152,7 @@
+ 
+   /* Double-check that the aSyscall[] array has been constructed
+   ** correctly.  See ticket [bb3a86e890c8e96ab] */
+-  assert( ArraySize(aSyscall)==24 );
++  assert( ArraySize(aSyscall)==25 );
+ 
+   /* Register all VFSes defined in the aVfs[] array */
+   for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
diff --git a/gnu/packages/sqlite.scm b/gnu/packages/sqlite.scm
index 8e976ce..26510c3 100644
--- a/gnu/packages/sqlite.scm
+++ b/gnu/packages/sqlite.scm
@@ -50,7 +50,9 @@
                     "/sqlite-autoconf-" numeric-version ".tar.gz")))
             (sha256
              (base32
-              "19gicv5vdi5c0p8shr1bmihldj409aqz3r4wr7d3pwb6xf1xv4p4"))))
+              "19gicv5vdi5c0p8shr1bmihldj409aqz3r4wr7d3pwb6xf1xv4p4"))
+            (patches
+             (list (search-patch "sqlite-large-page-size-fix.patch")))))
    (build-system gnu-build-system)
    (home-page "http://www.sqlite.org/")
    (synopsis "The SQLite database management system")
-- 
1.8.4

Re: [PATCHES] Upgrade sqlite and fix it on systems with page size > 32K

[Ludovic Courtès]

Mark H Weaver <mhw@netris.org> skribis:

> This is needed on systems with 64K page size.  It also upgrades sqlite
> to the latest version and computes the source URL automatically.

OK for both, thanks!

Ludo’.