From mboxrd@z Thu Jan 1 00:00:00 1970 Path: news.gmane.org!not-for-mail From: Mathias Dahl Newsgroups: gmane.emacs.help Subject: Re: Reading huge files Date: Wed, 10 Jan 2007 00:29:15 +0100 Message-ID: References: <8642ba650701081417n3c658d33v6dbf743a9bd30c7c@mail.gmail.com> NNTP-Posting-Host: lo.gmane.org Mime-Version: 1.0 Content-Type: text/plain; charset=us-ascii X-Trace: sea.gmane.org 1168386059 12170 80.91.229.12 (9 Jan 2007 23:40:59 GMT) X-Complaints-To: usenet@sea.gmane.org NNTP-Posting-Date: Tue, 9 Jan 2007 23:40:59 +0000 (UTC) Original-X-From: help-gnu-emacs-bounces+geh-help-gnu-emacs=m.gmane.org@gnu.org Wed Jan 10 00:40:57 2007 Return-path: Envelope-to: geh-help-gnu-emacs@m.gmane.org Original-Received: from lists.gnu.org ([199.232.76.165]) by lo.gmane.org with esmtp (Exim 4.50) id 1H4QaH-00052D-ER for geh-help-gnu-emacs@m.gmane.org; Wed, 10 Jan 2007 00:40:50 +0100 Original-Received: from localhost ([127.0.0.1] helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.43) id 1H4QaG-0003qw-S1 for geh-help-gnu-emacs@m.gmane.org; Tue, 09 Jan 2007 18:40:48 -0500 Original-Path: shelby.stanford.edu!newsfeed.stanford.edu!news.tele.dk!news.tele.dk!small.news.tele.dk!fu-berlin.de!uni-berlin.de!individual.net!not-for-mail Original-Newsgroups: gnu.emacs.help Original-Lines: 203 Original-X-Trace: individual.net qR3Bdyd10i4bZO5u2eeJsQhuMDyR+CrVRN1jwj2AEf61QZGUhM User-Agent: Gnus/5.11 (Gnus v5.11) Emacs/22.0.92 (gnu/linux) Cancel-Lock: sha1:K/FV/Uq+QznARz9POWbZb0VjlJw= Original-Xref: shelby.stanford.edu gnu.emacs.help:144598 Original-To: help-gnu-emacs@gnu.org X-BeenThere: help-gnu-emacs@gnu.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: Users list for the GNU Emacs text editor List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Original-Sender: help-gnu-emacs-bounces+geh-help-gnu-emacs=m.gmane.org@gnu.org Errors-To: help-gnu-emacs-bounces+geh-help-gnu-emacs=m.gmane.org@gnu.org Xref: news.gmane.org gmane.emacs.help:40203 Archived-At: Eli Zaretskii writes: > Does this really work? I think it won't: the underlying problem is > that for large files, END overflows the limits of the ELisp integer > type, and your suggestion doesn't resolve this fundamental problem. > So insert-file-contents will still barf with large files. > > Or am I missing something? I have only tested the code below with a 700 MB large movie file, and that is probably within the bounds you talk about. But even if it would fail (when does it fail?), being able to browse files hundreds of MB in size seems quite nice. Hence my new hack below. Enjoy! /Mathias ;;; vlf.el --- View Large Files ;; Copyright (C) 2006 Mathias Dahl ;; Version: 0.1 ;; Keywords: files, helpers, utilities ;; Author: Mathias Dahl ;; Maintainer: Mathias Dahl ;; URL: http://www.emacswiki.org/cgi-bin/wiki/VLF ;; This file is free software; you can redistribute it and/or modify ;; it under the terms of the GNU General Public License as published by ;; the Free Software Foundation; either version 2, or (at your option) ;; any later version. ;; This file is distributed in the hope that it will be useful, ;; but WITHOUT ANY WARRANTY; without even the implied warranty of ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;; GNU General Public License for more details. ;; You should have received a copy of the GNU General Public License ;; along with GNU Emacs; see the file COPYING. If not, write to ;; the Free Software Foundation, Inc., 59 Temple Place - Suite 330, ;; Boston, MA 02111-1307, USA. ;;; Commentary: ;; ;; After reading the Nth post on Gnu Emacs Help about Viewing Large ;; Files in Emacs, it itched so much that I decided to make a try. It ;; helped quite a lot when Kevin Rodgers posted a snippet on how to ;; use `insert-file-contents' to extract part of a file. At first I ;; made a try using head and tail and that worked too, but using ;; internal Emacs commands is nicer. Here is the code to extract data ;; using head and tail in case someone wanna try that out in the ;; future: ;; (defun vlf-extract-part-of-file (file from to) ;; "Returns bytes in FILE from FROM to TO." ;; (let ((size (vlf-file-size file))) ;; (if (or (> from size) ;; (> to size)) ;; (error "From or to is larger that the file size")) ;; (with-temp-buffer ;; (shell-command ;; (format "head --bytes %d %s | tail --bytes %d" ;; to file (+ (- to from) 1)) t) ;; (buffer-substring (point-min) (point-max))))) ;;; History: ;; ;; - Wed Jan 10 00:13:45 2007 ;; ;; First version created and released into the wild. ;;; Bugs ;; ;; Probably some. Feel free to fix them :) ;;; Code: (defgroup vlf nil "Browse large files in Emacs" :prefix "vlf-" :group 'files) (defcustom vlf-batch-size 1000 "Defines how large each batch of file data is." :type 'integer :group 'vlf) (defvar vlf-current-start-pos 1 "Keeps track of file position.") (defvar vlf-current-batch-size nil "Keeps track of current batch size.") (defvar vlf-current-file nil "File that is currently viewed.") (defvar vlf-mode-map (make-sparse-keymap) "Keymap for `vlf-mode'.") (defun vlf-define-keymap () "Define keymap for `vlf-mode'." (define-key vlf-mode-map [next] 'vlf-next) (define-key vlf-mode-map [prior] 'vlf-prev) (define-key vlf-mode-map "q" 'vlf-quit)) (define-derived-mode vlf-mode fundamental-mode "vlf-mode" "Mode to browse large files in. See `vlf' for details." (vlf-define-keymap) (toggle-read-only 1) (message "vlf-mode enabled")) (defun vlf-file-size (file) "Get size of FILE." (nth 7 (file-attributes file))) (defun vlf-quit () "Quit vlf." (interactive) (kill-buffer (current-buffer))) (defun vlf-insert-batch () "Insert current batch of data." (let* ((beg (1- vlf-current-start-pos)) (end (+ beg vlf-current-batch-size))) (insert-file-contents vlf-current-file nil beg end))) (defun vlf-next () "Display the next batch of file data." (interactive) (let ((inhibit-read-only t) left next-start-pos (size (vlf-file-size vlf-current-file))) (setq next-start-pos (+ vlf-current-start-pos vlf-batch-size)) (if (> next-start-pos size) (message "End of file") (setq vlf-current-batch-size vlf-batch-size vlf-current-start-pos next-start-pos left (1+ (- size vlf-current-start-pos))) (if (< left vlf-current-batch-size) (setq vlf-current-batch-size left)) (erase-buffer) (vlf-insert-batch) (rename-buffer (format "%s[%d,%d]" (file-name-nondirectory vlf-current-file) vlf-current-start-pos (1- (+ vlf-current-start-pos vlf-current-batch-size))))))) (defun vlf-prev () "Display the previous batch of file data." (interactive) (if (= 1 vlf-current-start-pos) (message "At beginning of file") (let ((inhibit-read-only t)) (erase-buffer) (setq vlf-current-start-pos (- vlf-current-start-pos vlf-batch-size) vlf-current-batch-size vlf-batch-size) (vlf-insert-batch) (rename-buffer (format "%s[%d,%d]" (file-name-nondirectory vlf-current-file) vlf-current-start-pos (1- (+ vlf-current-start-pos vlf-current-batch-size))))))) (defun vlf (file) "View a large file in Emacs FILE is the file to open. Batches of the file data from FILE will be displayed in a read-only buffer. You can customize the amount of bytes to display by customizing `vlf-batch-size'." (interactive "fFile to open: ") (setq vlf-current-file file vlf-current-start-pos 1 vlf-current-batch-size (1- (+ vlf-current-start-pos vlf-batch-size))) (switch-to-buffer (generate-new-buffer (format "%s[%d,%d]" (file-name-nondirectory file) vlf-current-start-pos (1- (+ vlf-current-start-pos vlf-current-batch-size))))) (erase-buffer) (vlf-insert-batch) (vlf-mode)) ;; (setq vlf-current-file "/home/mathias/movies/devil_in_a_blue_dress.avi") ;; (setq vlf-current-file "/home/mathias/extracttext") ;; (setq vlf-batch-size 3) ;; (setq vlf-current-start-pos 1) (provide 'vlf) ;;; vlf.el ends here