[daily][optimize] 一个小python程序的性能优化 （python类型转换函数引申的性能优化）

#! /usr/bin/python2import randomimport timedef main(): maxv = 1<<16 - 1 f = open('base.txt', 'w') random.seed(1476599333) for i in range(0, 600000): length = random.randint(10, 50) a = "" for i in range(0, length): v = random.randint(32, 126) a += chr(v) for i in range(0, 11): v = random.randint(0, maxv) a += ',' a += str(v) a += '\n' f.write(a) f.close()if '__main__' == __name__: main()

#! /usr/bin/python2def main(): f = open('base.txt', 'r') result = ['HOST', 0,0,0,0,0,0,0,0,0,0,0] for line in f.readlines(): l = line.strip().split(',') length = len(l) if length < 12 : raise Exception, "HUGE_TONG: Wrong format" skip = length - 12 for i in range(1,12): result[i] += int(l[i+skip]) f.close() print resultif __name__ == '__main__': r = main() exit(r)

[tong@T7 chimian]$ time ./run.py['HOST', 9829498952, 9833094366, 9827153757, 9835131709, 9843502266, 9836377986, 9825044768, 9833232152, 9841073437, 9833009489, 9833147503]real 0m3.787suser 0m3.747ssys 0m0.037s

[tong@T7 chimian]$ time ./run-md.py[, , , , , , , , , , ]real 0m1.208suser 0m1.180ssys 0m0.023s[tong@T7 chimian]$ diff run.py run-md.py13c13,< result[i] += int(l[i+skip])---> # result[i] += int(l[i+skip])> result[i] +=[tong@T7 chimian]$

>>> c=cProfile.Profile()>>> c.enable()>>> run.main()[, , , , , , , , , , ]>>> c.disable()>>> ps = pstats.Stats(c)>>> ps.sort_stats('cumulative').print_stats() function calls in 4.283 seconds Ordered by: cumulative time ncalls tottime percall cumtime percall filename:lineno(function) (<module>) (main) 0.318 0.000 0.318 0.000 {method 'split' of 'str' objects} 0.140 0.000 0.140 0.000 {range} 0.069 0.000 0.069 0.000 {method 'strip' of 'str' objects} 0.053 0.053 0.053 0.053 {method 'readlines' of 'file' objects} 0.033 0.000 0.033 0.000 {len} /encodings/utf_8.py:(decode) 0.000 0.000 0.000 0.000 {_codecs.utf_8_decode} 0.000 0.000 0.000 0.000 {open} 0.000 0.000 0.000 0.000 {method 'close' of 'file' objects} 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}<pstats.Stats instance at 0x7f6ef2c8e6c8>>>>

[tong@T7 ~]$ sudo pip2 install line_profiler

@profiledef main(): f = open('base.txt', 'r')

[tong@T7 chimian]$ kernprof -v -l run.py[, , , , , , , , , , ]Wrote profile results to run.py.lprofTimer unit: 1e- sTotal time: 10.9936 sFile: run.pyFunction: main at line Line # Hits Time Per Hit % Time Line Contents============================================================== @profile def main(): 8.0 0.0 f = open('base.txt', 'r') ,,,,,,,,,,] 0.5 2.9 for line in f.readlines(): 1.2 6.6 l = line.strip().split(',') 0.5 2.5 length = len(l) : raise Exception, "HUGE_TONG: Wrong format" ,): 0.9 54.2 result[i] += int(l[i+skip]) 14.0 0.0 f.close() 37.0 0.0 print result

[tong@T7 chimian]$ kernprof -v -l run.py[, , , , , , , , , , ]Wrote profile results to run.py.lprofTimer unit: 1e- sTotal time: 20.4831 sFile: run.pyFunction: main at line Line # Hits Time Per Hit % Time Line Contents============================================================== @profile def main(): 7.0 0.0 f = open('base.txt', 'r') ,,,,,,,,,,] 0.6 1.7 for line in f.readlines(): 1.3 3.8 l = line.strip().split(',') 0.5 1.5 length = len(l) : raise Exception, "HUGE_TONG: Wrong format" ,): # result[i] += int(l[i+skip]) 0.5 14.9 index = i + skip 0.5 14.7 value_str = l[index] 0.8 27.3 value_int = int(value_str) 0.5 16.8 result[i] += value_int 13.0 0.0 f.close() 37.0 0.0 print result

#include <stdio.h>#include <stdlib.h>#include <errno.h>#include <string.h>int main(){ FILE* fp = fopen("./base.txt", "r"); ); size_t len = ; ; ] = {,,,,,,,,,,}; ) { int r = getline(&buf, &len, fp); ) { ) perror("HUGE_TONG read(): "); break; } slen = strlen(buf); ] != '\n') { printf(]); } char* str; buf[slen-] = ; ; ; i >= ; i--) { if (buf[i] == ',') { str = buf + (i+); buf[i] = ; int value = atoi(str); array[index--] += value; ) { break; } } } } free(buf); fclose(fp); ; i < ; i++) { printf("%ld\t", array[i]); } printf("\n"); ;}

[tong@T7 chimian]$ gcc -Wall -O3 crun.c[tong@T7 chimian]$ time ./a.out real 0m0.288suser 0m0.277ssys 0m0.010s

[tong@T7 chimian]$ python2 -O3 -m py_compile run.py[tong@T7 chimian]$ time python2 run.pyo[, , , , , , , , , , ]real 0m3.865suser 0m3.833ssys 0m0.030s

[tong@T7 chimian]$ diff run.py run-md.py17c17< value_int = int(value_str)---> value_int =[tong@T7 chimian]$ vim run-md.py[tong@T7 chimian]$ vim run-md.py[tong@T7 chimian]$ time ./run-md.py[, , , , , , , , , , ]real 0m1.449suser 0m1.420ssys 0m0.027s[tong@T7 chimian]$ vim run-md.py[tong@T7 chimian]$ kernprof -v -l run-md.py[, , , , , , , , , , ]Wrote profile results to run-md.py.lprofTimer unit: 1e- sTotal time: 17.2632 sFile: run-md.pyFunction: main at line Line # Hits Time Per Hit % Time Line Contents============================================================== @profile def main(): 7.0 0.0 f = open('base.txt', 'r') ,,,,,,,,,,] 0.6 1.9 for line in f.readlines(): 1.2 4.2 l = line.strip().split(',') 0.5 1.7 length = len(l) : raise Exception, "HUGE_TONG: Wrong format" ,): # result[i] += int(l[i+skip]) 0.4 17.2 index = i + skip 0.4 17.0 value_str = l[index] 0.5 19.3 result[i] += value_int 12.0 0.0 f.close() 39.0 0.0 print result

[tong@T7 chimian]$ time ./run-md.py[, , , , , , , , , , ]real 0m0.960suser 0m0.923ssys 0m0.033s[tong@T7 chimian]$ cat run-md.py#! /usr/bin/python2#@profiledef main(): f = open('base.txt', 'r') result = [,,,,,,,,,,] for line in f.readlines(): l = line.strip().split(',') length = len(l) : raise Exception, "HUGE_TONG: Wrong format" skip = length - a = ,): # result[i] += int(l[i+skip]) # index = i + skip # value_str = l[index] # value_int = # result[i] += value_int # result[i] += a += f.close() print resultif __name__ == '__main__': r = main()# exit(r)[tong@T7 chimian]$

[tong@T7 chimian]$ time ./run-test.py real 0m0.372suser 0m0.293ssys 0m0.077s[tong@T7 chimian]$ cat run-test.py#! /usr/bin/python2def main(): a = ; , ): a += if __name__ == '__main__': main()[tong@T7 chimian]$