bug#74738: 31.0.50; Freezes in Python-mode on some Python file when searching or scrolling

[Eli Zaretskii <eliz@gnu.org>]

> From: rehan malak <rehan.malak@gmail.com>
> Date: Sun, 8 Dec 2024 14:05:36 +0100
> 
> I can reproduce systematically this freeze dealing with a 10000lines
> Python file :
> 
> wget https://raw.githubusercontent.com/hugsy/gef/refs/heads/main/gef.py
> emacs -Q gef.py

A much smaller reproducer is attached below.  Just visiting it freezes
Emacs.  But if I change the line marked below:

    @classmethod
    def is_valid(cls, _: pathlib.Path) -> bool:
        raise NotImplementedError

    def __str__(self) -> str:
  >>>>> return f"{self.name}('{self.path.absolute()}', entry @ {self.entry_point:#x})"

to say this instead:

        return self.name

the problem goes away.  So it is something in that complex expression
that trips syntax-ppss.

Stefan and kobarity, any suggestions or ideas?

> PageDown several times or scrolling with the mouse
> 
> Emacs freezes, Ctrl-g not working, CPU 100%
> 
> It works also by searching : Ctrl-s show RET
> then Ctrl-s several times
> 
> With a minimal .emacs :
> 
> (set debug-on-error t) 
> (set debug-on-quit t) 
> 
> and before scrolling
> 
> M-x profiler-start RET
> 
> then in an external terminal
> 
> pkill -SIGUSR2 emacs
> 
> I get the backtrace :
> 
> Debugger entered--entering a function:
> * #f(compiled-function () #<bytecode -0x179dcd1db31182ae>)()
>   syntax-ppss()
>   python-syntax-context-type()
>   python-nav-forward-block(-1)
>   python-nav-backward-block()
>   python-nav-beginning-of-block()
>   python-nav-end-of-block()
>   python-info-statement-ends-block-p()
>   python-info-end-of-block-p()
>   python-nav--forward-sexp(1 nil nil)
>   python-nav-forward-sexp(1)
>   forward-sexp(1)
>   up-list(1)
>   python--font-lock-f-strings(30419)
>   font-lock-fontify-keywords-region(28914 30419 nil)
>   font-lock-default-fontify-region(28914 30414 nil)
>   font-lock-fontify-region(28914 30414)
>   #f(compiled-function (fun) #<bytecode -0x8a96d65e1315e71>)(font-lock-fontify-region)
>   run-hook-wrapped(#f(compiled-function (fun) #<bytecode -0x8a96d65e1315e71>) font-lock-fontify-region)
>   jit-lock--run-functions(28914 30414)
>   jit-lock-fontify-now(28914 30414)
>   jit-lock-function(28914)

I see something different, in GDB:

  Lisp Backtrace:
  "parse-partial-sexp" (0x9f1a788)
  "syntax-ppss" (0x9f1a710)
  "python-info-line-ends-backslash-p" (0x9f1a6b8)
  "python-nav-end-of-statement" (0x9f1a670)
  "python-nav-end-of-block" (0x9f1a630)
  "python-info-statement-ends-block-p" (0x9f1a610)
  "python-info-end-of-block-p" (0x9f1a5c0)
  "python-nav--forward-sexp" (0x9f1a548)
  "python-nav-forward-sexp" (0x9f1a4f8)
  "forward-sexp" (0x9f1a4a8)
  "up-list" (0x9f1a438)
  "python--font-lock-f-strings" (0x9f1a3a0)
  "font-lock-fontify-keywords-region" (0x9f1a308)
  "font-lock-default-fontify-region" (0x9f1a2a0)
  "font-lock-fontify-region" (0x9f1a230)
  0xb9117d0 PVEC_CLOSURE
  "run-hook-wrapped" (0x9f1a1c0)
  "jit-lock--run-functions" (0x9f1a0e8)
  "jit-lock-fontify-now" (0x9f1a058)
  "jit-lock-function" (0x5ffba98)
  "redisplay_internal (C function)" (0x0)

Here's the file with which I can reproduce the problem?


class FileFormat:
    name: str
    path: pathlib.Path
    entry_point: int
    checksec: dict[str, bool]
    sections: list[FileFormatSection]

    def __init__(self, path: str | pathlib.Path) -> None:
        raise NotImplementedError

    def __init_subclass__(cls: Type["FileFormat"], **kwargs):
        global __registered_file_formats__
        super().__init_subclass__(**kwargs)
        required_attributes = ("name", "entry_point", "is_valid", "checksec",)
        for attr in required_attributes:
            if not hasattr(cls, attr):
                raise NotImplementedError(f"File format '{cls.__name__}' is invalid: missing attribute '{attr}'")
        __registered_file_formats__.add(cls)
        return

    @classmethod
    def is_valid(cls, _: pathlib.Path) -> bool:
        raise NotImplementedError

    def __str__(self) -> str:
        return f"{self.name}('{self.path.absolute()}', entry @ {self.entry_point:#x})"


class Elf(FileFormat):
    """Basic ELF parsing.
    Ref:
    - http://www.skyfree.org/linux/references/ELF_Format.pdf
    - https://refspecs.linuxfoundation.org/elf/elfspec_ppc.pdf
    - https://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html
    """
    class Class(enum.Enum):
        ELF_32_BITS       = 0x01
        ELF_64_BITS       = 0x02

    ELF_MAGIC         = 0x7f454c46

    class Abi(enum.Enum):
        X86_64            = 0x3e
        X86_32            = 0x03
        ARM               = 0x28
        MIPS              = 0x08
        POWERPC           = 0x14
        POWERPC64         = 0x15
        SPARC             = 0x02
        SPARC64           = 0x2b
        AARCH64           = 0xb7
        RISCV             = 0xf3
        IA64              = 0x32
        M68K              = 0x04

    class Type(enum.Enum):
        ET_RELOC          = 1
        ET_EXEC           = 2
        ET_DYN            = 3
        ET_CORE           = 4

    class OsAbi(enum.Enum):
        SYSTEMV     = 0x00
        HPUX        = 0x01
        NETBSD      = 0x02
        LINUX       = 0x03
        SOLARIS     = 0x06
        AIX         = 0x07
        IRIX        = 0x08
        FREEBSD     = 0x09
        OPENBSD     = 0x0C

    e_magic: int                = ELF_MAGIC
    e_class: "Elf.Class"        = Class.ELF_32_BITS
    e_endianness: Endianness    = Endianness.LITTLE_ENDIAN
    e_eiversion: int
    e_osabi: "Elf.OsAbi"
    e_abiversion: int
    e_pad: bytes
    e_type: "Elf.Type"          = Type.ET_EXEC
    e_machine: Abi              = Abi.X86_32
    e_version: int
    e_entry: int
    e_phoff: int
    e_shoff: int
    e_flags: int
    e_ehsize: int
    e_phentsize: int
    e_phnum: int
    e_shentsize: int
    e_shnum: int
    e_shstrndx: int

    path: pathlib.Path
    phdrs : list["Phdr"]
    shdrs : list["Shdr"]
    name: str = "ELF"

    __checksec : dict[str, bool]

    def __init__(self, path: str | pathlib.Path) -> None:
        """Instantiate an ELF object. A valid ELF must be provided, or an exception will be thrown."""

        if isinstance(path, str):
            self.path = pathlib.Path(path).expanduser()
        elif isinstance(path, pathlib.Path):
            self.path = path
        else:
            raise TypeError

        if not self.path.exists():
            raise FileNotFoundError(f"'{self.path}' not found/readable, most gef features will not work")

        self.__checksec = {}

        with self.path.open("rb") as self.fd:
            # off 0x0
            self.e_magic, e_class, e_endianness, self.e_eiversion = self.read_and_unpack(">IBBB")
            if self.e_magic != Elf.ELF_MAGIC:
                # The ELF is corrupted, GDB won't handle it, no point going further
                raise RuntimeError("Not a valid ELF file (magic)")

            self.e_class, self.e_endianness = Elf.Class(e_class), Endianness(e_endianness)

            if self.e_endianness != gef.arch.endianness:
                warn("Unexpected endianness for architecture")

            endian = self.e_endianness

            # off 0x7
            e_osabi, self.e_abiversion = self.read_and_unpack(f"{endian}BB")
            self.e_osabi = Elf.OsAbi(e_osabi)

            # off 0x9
            self.e_pad = self.read(7)

            # off 0x10
            e_type, e_machine, self.e_version = self.read_and_unpack(f"{endian}HHI")
            self.e_type, self.e_machine = Elf.Type(e_type), Elf.Abi(e_machine)

            # off 0x18
            if self.e_class == Elf.Class.ELF_64_BITS:
                self.e_entry, self.e_phoff, self.e_shoff = self.read_and_unpack(f"{endian}QQQ")
            else:
                self.e_entry, self.e_phoff, self.e_shoff = self.read_and_unpack(f"{endian}III")

            self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum = self.read_and_unpack(f"{endian}IHHH")
            self.e_shentsize, self.e_shnum, self.e_shstrndx = self.read_and_unpack(f"{endian}HHH")

            self.phdrs = []
            for i in range(self.e_phnum):
                self.phdrs.append(Phdr(self, self.e_phoff + self.e_phentsize * i))

            self.shdrs = []
            for i in range(self.e_shnum):
                self.shdrs.append(Shdr(self, self.e_shoff + self.e_shentsize * i))
        return

    def read(self, size: int) -> bytes:
        return self.fd.read(size)

    def read_and_unpack(self, fmt: str) -> tuple[Any, ...]:
        size = struct.calcsize(fmt)
        data = self.fd.read(size)
        return struct.unpack(fmt, data)

    def seek(self, off: int) -> None:
        self.fd.seek(off, 0)

    def __str__(self) -> str:
        return f"ELF('{self.path.absolute()}', {self.e_class.name}, {self.e_machine.name})"

    def __repr__(self) -> str:
        return f"ELF('{self.path.absolute()}', {self.e_class.name}, {self.e_machine.name})"

    @property
    def entry_point(self) -> int:
        return self.e_entry

    @classmethod
    def is_valid(cls, path: pathlib.Path) -> bool:
        return u32(path.open("rb").read(4), e = Endianness.BIG_ENDIAN) == Elf.ELF_MAGIC

    @property
    def checksec(self) -> dict[str, bool]:
        """Check the security property of the ELF binary. The following properties are:
        - Canary
        - NX
        - PIE
        - Fortify
        - Partial/Full RelRO.
        Return a dict() with the different keys mentioned above, and the boolean
        associated whether the protection was found."""
        if not self.__checksec:
            def __check_security_property(opt: str, filename: str, pattern: str) -> bool:
                cmd   = [readelf,]
                cmd  += opt.split()
                cmd  += [filename,]
                lines = gef_execute_external(cmd, as_list=True)
                for line in lines:
                    if re.search(pattern, line):
                        return True
                return False

            abspath = str(self.path.absolute())
            readelf = gef.session.constants["readelf"]
            self.__checksec["Canary"] = __check_security_property("-rs", abspath, r"__stack_chk_fail") is True
            has_gnu_stack = __check_security_property("-W -l", abspath, r"GNU_STACK") is True
            if has_gnu_stack:
                self.__checksec["NX"] = __check_security_property("-W -l", abspath, r"GNU_STACK.*RWE") is False
            else:
                self.__checksec["NX"] = False
            self.__checksec["PIE"] = __check_security_property("-h", abspath, r":.*EXEC") is False
            self.__checksec["Fortify"] = __check_security_property("-s", abspath, r"_chk@GLIBC") is True
            self.__checksec["Partial RelRO"] = __check_security_property("-l", abspath, r"GNU_RELRO") is True
            self.__checksec["Full RelRO"] = self.__checksec["Partial RelRO"] and __check_security_property("-d", abspath, r"BIND_NOW") is True
        return self.__checksec

    @classproperty
    @deprecated("use `Elf.Abi.X86_64`")
    def X86_64(cls) -> int: return Elf.Abi.X86_64.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.X86_32`")
    def X86_32(cls) -> int : return Elf.Abi.X86_32.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.ARM`")
    def ARM(cls) -> int : return Elf.Abi.ARM.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.MIPS`")
    def MIPS(cls) -> int : return Elf.Abi.MIPS.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.POWERPC`")
    def POWERPC(cls) -> int : return Elf.Abi.POWERPC.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.POWERPC64`")
    def POWERPC64(cls) -> int : return Elf.Abi.POWERPC64.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.SPARC`")
    def SPARC(cls) -> int : return Elf.Abi.SPARC.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.SPARC64`")
    def SPARC64(cls) -> int : return Elf.Abi.SPARC64.value # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.AARCH64`")
    def AARCH64(cls) -> int : return Elf.Abi.AARCH64.value  # pylint: disable=no-self-argument

    @classproperty
    @deprecated("use `Elf.Abi.RISCV`")
    def RISCV(cls) -> int : return Elf.Abi.RISCV.value # pylint: disable=no-self-argument