Popen encountered enomen failure of the bug survey

Problem Description

The company's original program recently encountered an interesting problem, the system has memory 3T, memory consumption 2.5T, this time the log shows open gzip format file failed. The program is the old-fashioned C program, and the way to open gzip files is this:

FD = Popen ("Gzip-dc xxx.gz", "R");

I added a print errno in the back and found this error to be Enomem (12). Problem Research

Query keyword Popen +enomem, all of a sudden a lot of posts are put forward is the popen process of clone call in the replication of the parent process when the failure, feel some truth, and not so reasonable, it is reasonable if you really try to replicate the parent process completely, Then because of the parent process and the majority of the system's memory, there is absolutely no possibility of replication success, it makes no sense that the Linux process replication should be copy-on-write, the process is copied immediately after the shell to perform gzip, memory consumption should not be very good.

A Linux virtual machine with 5G of memory on hand, write down the following code test;
I use 4G memory first and then try to read a file in gzip format using the interface of the Popen or zlib library, respectively.

#include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include < iostream> #include <zlib.h> int main () {#define SIZE 4000000000L #define USE_ZLIB 0 char * used = (char *)
    malloc (sizeof (char) *size);
    for (long i = 0; i< SIZE; i++) {Used[i] = ' 1 '; } std::cout<< "1!"
    <<std::endl;
    Sleep (1); std::cout<< "2!"
    <<std::endl;
        if (use_zlib) {gzfile fp = Gzopen ("test.gz", "R"); if (fp = = NULL) {std::cout<< "Zlib err!"
        <<std::endl; } else {std::cout<< "zlib corr!"
            <<std::endl;
            Char buff[1024];
            while (Gzgets (fp,buff,1024)! = NULL) {std::cout<<buff;
        }}} else {FILE * fp = Popen ("Gzip-dc test.gz", "R");
        Sleep (1); std::cout<< "3!"
<<std::endl;        if (fp = = NULL) {std::cout<< "Err!"
        <<std::endl; } else {std::cout<< "corr!"
            <<std::endl;
            Char buff[1024];
            while (Fgets (BUFF,1024,FP)! = NULL) {std::cout<<buff;

}}} return 0; }

Compile command:

g++-G Test.cpp-lz

Sure enough, the use of Popen fails, but the USE_ZLIB macro is set to 1 to run.

./a.out
1!
2!
3!
Err!

Perform

Strace./a.out
Execve ("./a.out", ["./a.out"], [/* VARs */]) = 0
BRK (0) = 0x15b7000
Mmap (NULL, 4096, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c1895000
Access ("/etc/ld.so.preload", R_OK) =-1 ENOENT (No such file or directory)
Open ("/etc/ld.so.cache", o_rdonly) = 3
Fstat (3, {st_mode=s_ifreg|0644, st_size=40201, ...}) = 0
Mmap (NULL, 40201, Prot_read, Map_private, 3, 0) = 0x7f24c188b000
Close (3) = 0
Open ("/usr/lib64/libstdc++.so.6", o_rdonly) = 3
Read (3, "\177elf\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360c\5\0\0\0\0\0" ..., 832) = 832
Fstat (3, {st_mode=s_ifreg|0755, st_size=987096, ...}) = 0
Mmap (NULL, 3166648, prot_read| Prot_exec, map_private| Map_denywrite, 3, 0) = 0x7f24c1371000
Mprotect (0x7f24c1459000, 2097152, prot_none) = 0
Mmap (0x7f24c1659000, 36864, prot_read| Prot_write, map_private| map_fixed| Map_denywrite, 3, 0xe8000) = 0x7f24c1659000
Mmap (0x7f24c1662000, 82360, prot_read| Prot_write, map_private| map_fixed| Map_anonymous,-1, 0) = 0x7f24c1662000
Close (3) = 0
Open ("/lib64/libm.so.6", o_rdonly) = 3
Read (3, "\177elf\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p>\0\0\0\0\0\0" ..., 832) = 832
Fstat (3, {st_mode=s_ifreg|0755, st_size=596360, ...}) = 0
Mmap (NULL, 2633912, prot_read| Prot_exec, map_private| Map_denywrite, 3, 0) = 0x7f24c10ed000
Mprotect (0x7f24c1170000, 2093056, prot_none) = 0
Mmap (0x7f24c136f000, 8192, prot_read| Prot_write, map_private| map_fixed| Map_denywrite, 3, 0x82000) = 0x7f24c136f000
Close (3) = 0
Open ("/lib64/libgcc_s.so.1", o_rdonly) = 3
Read (3, "\177elf\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20) \0\0\0\0\0\0" ..., 832) = 832
Fstat (3, {st_mode=s_ifreg|0755, st_size=90880, ...}) = 0
Mmap (NULL, 4096, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c188a000
Mmap (NULL, 2186584, prot_read| Prot_exec, map_private| Map_denywrite, 3, 0) = 0x7f24c0ed7000
Mprotect (0x7f24c0eed000, 2093056, prot_none) = 0
Mmap (0x7f24c10ec000, 4096, prot_read| Prot_write, map_private| map_fixed| Map_denywrite, 3, 0x15000) = 0x7f24c10ec000
Close (3) = 0
Open ("/lib64/libc.so.6", o_rdonly) = 3
Read (3, "\177elf\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\356\1\0\0\0\0\0" ..., 832) = 832
Fstat (3, {st_mode=s_ifreg|0755, st_size=1923352, ...}) = 0
Mmap (NULL, 3750184, prot_read| Prot_exec, map_private| Map_denywrite, 3, 0) = 0x7f24c0b43000
Mprotect (0x7f24c0ccd000, 2097152, prot_none) = 0
Mmap (0x7f24c0ecd000, 24576, prot_read| Prot_write, map_private| map_fixed| Map_denywrite, 3, 0x18a000) = 0x7f24c0ecd000
Mmap (0x7f24c0ed3000, 14632, prot_read| Prot_write, map_private| map_fixed| Map_anonymous,-1, 0) = 0x7f24c0ed3000
Close (3) = 0
Mmap (NULL, 4096, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c1889000
Mmap (NULL, 8192, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c1887000
Arch_prctl (ARCH_SET_FS, 0x7f24c1887720) = 0
Mprotect (0x7f24c0ecd000, 16384, prot_read) = 0
Mprotect (0x7f24c136f000, 4096, prot_read) = 0
Mmap (NULL, 4096, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c1886000
Mprotect (0x7f24c1659000, 28672, prot_read) = 0
Mprotect (0x7f24c1896000, 4096, prot_read) = 0
Munmap (0x7f24c188b000, 40201) = 0
Mmap (NULL, 1100001280, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f247f238000
Fstat (1, {st_mode=s_ifchr|0620, St_rdev=makedev (136, 1), ...}) = 0
Mmap (NULL, 4096, prot_read| Prot_write, map_private| Map_anonymous,-1, 0) = 0x7f24c1894000
Write (1, "1!\n", 4 1!)
) = 4
Rt_sigprocmask (Sig_block, [chld], [], 8) = 0
Rt_sigaction (SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
Rt_sigprocmask (Sig_setmask, [], NULL, 8) = 0
Nanosleep ({1, 0}, 0x7ffd9908b5f0) = 0
Write (1, "2!\n", 4 2!)
) = 4
BRK (0) = 0x15b7000
BRK (0x15d8000) = 0x15d8000
Pipe2 ([3, 4], o_cloexec) = 0

Clone (Child_stack=0, flags=clone_child_cleartid| clone_child_settid| SIGCHLD, child_tidptr=0x7f24c18879f0) =-1 Enomem (Cannot allocate memory)
Close (4) = 0
Close (3) = 0
Rt_sigprocmask (Sig_block, [chld], [], 8) = 0
Rt_sigaction (SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
Rt_sigprocmask (Sig_setmask, [], NULL, 8) = 0

Nanosleep ({1, 0}, 0x7ffd9908b5f0) = 0
Write (1, "3!\n", 4 3!)
) = 4
Write (1, "Err!\n", 7 err!)
) = 7
Exit_group (0) =?
+ + + exited with 0 + + + +

This is not bad, although it is not sure how much memory is required when cloning, but it should be with the parent process memory consumption has a certain relationship. It is not recommended to use Popen when the parent process is consuming too much memory.
Here I change directly with Zlib library, but if must use Popen, Netizen provides the following solution:

Use Vfork (no clone) to write a function instead of Popen

#ifdef Open_max//static Long OpenMAX = Open_max;
#else static Long OpenMAX = 0;
 #endif/* * If Open_max is indeterminate, we ' re isn't * guaranteed that the is adequate. */#define Open_max_guess Open_max (void) {if (OpenMAX = = 0) {/* First time through */errno
        = 0;    if ((OpenMAX = sysconf (_sc_open_max)) < 0) {if (errno = = 0) OpenMAX = open_max_guess;
        /* it ' s indeterminate */else printf ("Sysconf error for _sc_open_max");
}} return (OpenMAX);  } static pid_t *childpid = NULL;  /* ptr to array allocated at Run-time */static int maxfd;
    /* from our Open_max (), {Prog OpenMAX} */FILE *vpopen (const char* cmdstring, const char *type) {int pfd[2];
    FILE *FP;

    pid_t pid; if ((type[0]!= ' R ' && type[0]!= ' W ') | |
        type[1]!=0) {errno = EINVAL;
    return (NULL);  
   if (childpid = = NULL) {/* First time through */             /* Allocate zeroed out array for child PIDs */maxfd = Open_max ();  
    if (Childpid = (pid_t *) calloc (maxfd, sizeof (pid_t)) = = null) return (NULL);
    } if (pipe (PFD)!=0) {return NULL;   } if (PID = Vfork ()) <0) {return (NULL);
            /* errno set by fork () */} else if (PID = = 0) {/* Child */if (*type = = ' R ') {  
            Close (pfd[0]);  
                if (pfd[1]! = Stdout_fileno) {dup2 (pfd[1], Stdout_fileno);  
            Close (pfd[1]);  
            }} else {close (pfd[1]);  
                if (pfd[0]! = Stdin_fileno) {dup2 (pfd[0], Stdin_fileno);  
            Close (pfd[0]);  
        }}/* Close all descriptors in childpid[] */for (int i = 0; i < maxfd; i++)  

        if (childpid[i] > 0) Close (i); Execl ("/Bin/sh "," sh ","-C ", Cmdstring, (char *) 0);     
    _exit (127);  
        } if (*type = = ' R ') {Close (pfd[1]);  
    if (fp = Fdopen (pfd[0], type) = = null) return (NULL);  
        } else {close (pfd[0]);  
    if (fp = Fdopen (pfd[1], type) = = null) return (NULL); } childpid[fileno (fp)] = pid;     
/* Remember child pid for this FD */return (FP);  
    } int Vpclose (FILE *fp) {int FD, stat;  

    pid_t pid;     if (childpid = = NULL) return (-1);  
    /* Popen () has never been called */FD = Fileno (FP);     if (PID = childpid[fd]) = = 0) return (-1);  
    /* FP wasn ' t opened by Popen () */CHILDPID[FD] = 0;  

    if (fclose (fp) = = EOF) return (-1); while (Waitpid (PID, &stat, 0) < 0) if (errno! = eintr) return (-1);   /* ERROR other than eintr from Waitpid () */return (STAT);  /* return child ' s termination status */

}