first commit

README.md

@@ -0,0 +1,2 @@
+# util
+

authen/__init__.py

@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

authen/util/encrypt.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+import hashlib
+
+def sort_hash_urlencode(d, pop_keys=None):
+    """对字典进行排序, 按照key的顺序从小到大进行排列"""
+    d2 = d.copy()
+    # 剔除不参与排序的key
+    if pop_keys is not None:
+        for k in pop_keys:
+            if k in d2:
+                d2.pop(k)
+    return "&".join([
+        "%s=%s" % i for i in 
+        sorted(d2.items(), key=lambda m:m[0])
+    ])
+
+def create_hexdigest_sign(data={}, secret_key="", sign_key="sign"):
+    """对请求参数进行签名"""
+    if secret_key == "":
+        raise Exception("secret_key is required.")
+    if not data:
+        raise Exception("data is required.")
+
+    request_str = sort_hash_urlencode(data, pop_keys=(sign_key,))
+    unsigned_str = "%s%s" % (request_str, secret_key)
+    return hashlib.md5(unsigned_str).hexdigest()
+
+def create_hexdigest_data(data={}, secret_key="", sign_key="sign"):
+    """生成签名参数"""
+    sign_val = create_hexdigest_sign(
+        data=data, 
+        secret_key=secret_key, 
+        sign_key=sign_key)
+    data.update({sign_key: sign_val})
+    return data
+
+def validate_hexdigest_data(data={}, secret_key="", sign_key="sign"):
+    """验证签名"""
+    if sign_key not in data:
+        raise Exception("sign is required.")
+    req_sign_val = data[sign_key]
+    sign_val = create_hexdigest_sign(
+        data=data, 
+        secret_key=secret_key, 
+        sign_key=sign_key)
+    return req_sign_val == sign_val

authen/util/httpreq/__init__.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+import os
+import ssl
+import gzip
+import StringIO
+import datetime
+import urllib
+import urllib2
+import simplejson as json
+from traceback import format_exc
+
+
+def send_req(url, postdata=None, json_req=False, headers={}, proxy_host=''):
+    """
+    发送http请求，
+    postdata不为None时，发送post请求
+    postdata为None时，发送get请求
+    """
+    print datetime.datetime.now()
+    if proxy_host:
+        print proxy_host
+    if os.environ.get('http_proxy', '') != proxy_host:
+        os.environ['http_proxy'] = proxy_host
+        proxy = urllib2.ProxyHandler({'http': proxy_host})
+        opener = urllib2.build_opener(proxy)
+        urllib2.install_opener(opener)
+        # print os.environ
+    else:
+        if 'http_proxy' in os.environ:
+            del os.environ['http_proxy']
+    print url, postdata
+
+    if postdata:
+        if json_req:
+            js_data = json.dumps(postdata, ensure_ascii=False).encode("utf8")
+            req = urllib2.Request(url, js_data)
+            req.add_header('Content-Type', 'application/json')
+        else:
+            req = urllib2.Request(url, urllib.urlencode(postdata))
+    else:
+        req = urllib2.Request(url)
+    if headers:
+        print headers
+        req.headers = headers
+    req.add_header('encoding', 'utf-8')
+
+    try:
+        if url.startswith("https"):
+            context = ssl._create_unverified_context()
+            response = urllib2.urlopen(req, context=context, timeout=20)
+        else:
+            response = urllib2.urlopen(req, timeout=10)
+        resp_encoding = response.headers.get('Content-Encoding', '')
+        if resp_encoding == 'gzip':
+            gz_content = response.read()
+            compressedstream = StringIO.StringIO(gz_content)
+            gzipper = gzip.GzipFile(fileobj=compressedstream)
+            resp = gzipper.read()
+        else:
+            resp = response.read()
+            resp_type = response.headers.get('Content-Type', '')
+        if "json" in resp_type:
+            resp = json.loads(resp)
+
+    except urllib2.URLError, e:
+        error_info = format_exc()
+        return error_info
+    except Exception, e:
+        error_info = format_exc()
+        print e
+        print error_info
+        return error_info
+    print datetime.datetime.now()
+    return resp
+

authen/util/ipip.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from traceback import format_exc
+import struct
+from socket import inet_aton
+import os
+
+_unpack_V = lambda b: struct.unpack("<L", b)
+_unpack_N = lambda b: struct.unpack(">L", b)
+_unpack_C = lambda b: struct.unpack("B", b)
+
+
+class IP:
+    offset = 0
+    index = 0
+    binary = ""
+
+    @staticmethod
+    def load(file):
+        try:
+            path = os.path.abspath(file)
+            with open(path, "rb") as f:
+                IP.binary = f.read()
+                IP.offset, = _unpack_N(IP.binary[:4])
+                IP.index = IP.binary[4:IP.offset]
+        except Exception as ex:
+            print "cannot open file %s" % file
+            print format_exc()
+            exit(0)
+
+    @staticmethod
+    def find(ip):
+        index = IP.index
+        offset = IP.offset
+        binary = IP.binary
+        nip = inet_aton(ip)
+        ipdot = ip.split('.')
+        if int(ipdot[0]) < 0 or int(ipdot[0]) > 255 or len(ipdot) != 4:
+            return "N/A"
+
+        tmp_offset = int(ipdot[0]) * 4
+        start, = _unpack_V(index[tmp_offset:tmp_offset + 4])
+
+        index_offset = index_length = 0
+        max_comp_len = offset - 1028
+        start = start * 8 + 1024
+        while start < max_comp_len:
+            if index[start:start + 4] >= nip:
+                index_offset, = _unpack_V(index[start + 4:start + 7] + chr(0).encode('utf-8'))
+                index_length, = _unpack_C(index[start + 7])
+                break
+            start += 8
+
+        if index_offset == 0:
+            return "N/A"
+
+        res_offset = offset + index_offset - 1024
+        return binary[res_offset:res_offset + index_length].decode('utf-8')
+
+
+class IPX:
+    binary = ""
+    index = 0
+    offset = 0
+
+    @staticmethod
+    def load(file):
+        try:
+            path = os.path.abspath(file)
+            with open(path, "rb") as f:
+                IPX.binary = f.read()
+                IPX.offset, = _unpack_N(IPX.binary[:4])
+                IPX.index = IPX.binary[4:IPX.offset]
+        except Exception as ex:
+            print "cannot open file %s" % file
+            print ex.message
+            exit(0)
+
+    @staticmethod
+    def find(ip):
+        index = IPX.index
+        offset = IPX.offset
+        binary = IPX.binary
+        nip = inet_aton(ip)
+        ipdot = ip.split('.')
+        if int(ipdot[0]) < 0 or int(ipdot[0]) > 255 or len(ipdot) != 4:
+            return "N/A"
+
+        tmp_offset = (int(ipdot[0]) * 256 + int(ipdot[1])) * 4
+        start, = _unpack_V(index[tmp_offset:tmp_offset + 4])
+
+        index_offset = index_length = -1
+        max_comp_len = offset - 262144 - 4
+        start = start * 9 + 262144
+
+        while start < max_comp_len:
+            if index[start:start + 4] >= nip:
+                index_offset, = _unpack_V(index[start + 4:start + 7] + chr(0).encode('utf-8'))
+                index_length, = _unpack_C(index[start + 8:start + 9])
+                break
+            start += 9
+
+        if index_offset == 0:
+            return "N/A"
+
+        res_offset = offset + index_offset - 262144
+        return binary[res_offset:res_offset + index_length].decode('utf-8')

authen/util/ipv4.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import sys
+reload(sys)
+sys.setdefaultencoding("utf-8")
+from ipip import IP
+from ipip import IPX
+
+
+IP.load("/data/db/mydata4vipday2.dat")

authen/util/pagination.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+
+from math import ceil
+
+from sqlalchemy import text
+
+
+def slice_list(list, slice_into):
+    for i in xrange(0, len(list), slice_into):
+        yield list[i:i + slice_into]
+
+class BasePages(object):
+
+    def __init__(self, Query, page, per_page):
+        self.page = page
+        self.per_page = per_page
+        self.Query = Query
+        self.total_count = self._load_count()
+
+    def _load_count(self):
+        raise NotImplementedError()
+
+    @property
+    def pages(self):
+        return int(ceil(self.total_count / float(self.per_page)))
+
+    @property
+    def has_prev(self):
+        return self.page > 1
+
+    @property
+    def has_next(self):
+        return self.page < self.pages
+
+    def iter_pages(self, left_edge=2, left_current=2, right_current=5, right_edge=2):
+        last = 0
+        for num in xrange(1, self.pages + 1):
+            if num <= left_edge or \
+               (num > self.page - left_current - 1 and \
+                num < self.page + right_current) or \
+               num > self.pages - right_edge:
+                if last + 1 != num:
+                    yield None
+                yield num
+                last = num
+
+    def list(self):
+        raise NotImplementedError()
+
+class SqlalchemyPages(BasePages):
+    def _load_count(self):
+        return self.Query.count()
+
+    def list(self):
+        return self.Query.offset((self.page-1)*self.per_page)\
+                .limit(self.per_page).all()
+
+class SqlPages(BasePages):
+    def __init__(self, Query, page, per_page, db, **query_args):
+        self.query_args = query_args
+        self.db = db
+        super(SqlPages, self).__init__(
+            self._remove_dotted(Query), page, per_page)
+
+    def _remove_dotted(self, sql):
+        return sql.strip()[:-1] if sql.strip().endswith(";") else sql
+
+    def sum(self, *group_args):
+        group_sql = "SELECT %s FROM (%s) T;" % (
+                ", ".join([ "SUM(%s) %s" % (x, x) for x in group_args ]),
+                self.Query
+            )
+        record = self.db.session.execute(text(group_sql), self.query_args)
+        rv = []
+        for row in record.fetchall():
+            row_dict = dict(zip(row.keys(), row))
+            rv.append(row_dict)
+        return rv[0]
+
+    def _load_count(self):
+        count_sql = "SELECT COUNT(*) c FROM (%s) t;" % self.Query
+        record = self.db.session.execute(text(count_sql), self.query_args)
+        count = record.fetchone()
+        return count[0]
+
+    def list(self):
+        query_sql = "%s offset %s limit %s;" % (
+            self.Query,
+            (self.page-1)*self.per_page,
+            self.per_page
+        )
+        record = self.db.session.execute(text(query_sql), self.query_args)
+        rv = []
+        for row in record.fetchall():
+            row_dict = dict(zip(row.keys(), row))
+            rv.append(row_dict)
+        return rv
+
+class ListPages(BasePages):
+    def _load_count(self):
+        return len(self.Query)
+
+    def list(self):
+        object_list = []
+
+        for list in slice_list(self.Query, self.per_page):
+            object_list.append(list)
+
+        if not object_list:
+            object_list = [[]]
+        try:
+            result = object_list[self.page-1]
+        except IndexError:
+            result = object_list[0]
+        return result
+
+def object_list(Query, per_page, page):
+    
+    object_list = []
+
+    for list in slice_list(Query, per_page):
+        object_list.append(list)
+
+    if not object_list:
+        object_list = [[]]
+    try:
+        result = object_list[page-1]
+    except IndexError:
+        result = object_list[0]
+    return result
+
+class Pages(object):
+    def __init__(self, Query, page, per_page):
+        self.page = page
+        self.per_page = per_page
+        try:
+            self.total_count = Query.row_count
+        except Exception as e:
+            self.total_count = Query
+
+    @property
+    def pages(self):
+        return int(ceil(self.total_count / float(self.per_page)))
+
+
+    @property
+    def has_prev(self):
+        return self.page > 1
+
+
+    @property
+    def has_next(self):
+        return self.page < self.pages
+
+
+    def iter_pages(self, left_edge=2, left_current=2,
+                   right_current=5, right_edge=2):
+        last = 0
+        for num in xrange(1, self.pages + 1):
+            if num <= left_edge or \
+                    (num > self.page - left_current - 1 and \
+                                 num < self.page + right_current) or \
+                            num > self.pages - right_edge:
+                if last + 1 != num:
+                    yield None
+                yield num
+                last = num

authen/util/password.py

@@ -0,0 +1,47 @@
+# -*- coding:utf-8 -*-
+
+import sys
+import random
+from functools import wraps
+if sys.version_info >= (2, 5):
+    import hashlib
+    md5_constructor = hashlib.md5
+    md5_hmac = md5_constructor
+    sha_constructor = hashlib.sha1
+    sha_hmac = sha_constructor
+else:
+    import md5
+    md5_constructor = md5.new
+    md5_hmac = md5
+    import sha
+    sha_constructor = sha.new
+    sha_hmac = sha
+
+def get_hexdigest(algorithm, salt, raw_password):
+    """
+    Returns a string of the hexdigest of the given plaintext password and salt
+    using the given algorithm ('md5', 'sha1' or 'crypt').
+    """
+    
+    if algorithm == 'crypt':
+        try:
+            import crypt
+        except ImportError:
+            raise ValueError('"crypt" password algorithm not supported in this environment')
+        return crypt.crypt(raw_password, salt)
+
+    if algorithm == 'md5':
+        return md5_constructor(salt + raw_password).hexdigest()
+    elif algorithm == 'sha1':
+        return sha_constructor(salt + raw_password).hexdigest()
+    raise ValueError("Got unknown password algorithm type in password.")
+
+def get_salt(length=5):  
+    "获取由4位随机大小写字母、数字组成的salt值"
+    salt = ""  
+    chars = "AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789"  
+    len_chars = len(chars) - 1  
+    r = random.Random()  
+    for i in xrange(length):  
+        salt += chars[r.randint(0, len_chars)]  
+    return salt  

authen/util/string.py

@@ -0,0 +1,4 @@
+# -*- coding:utf-8 -*-
+
+def filter_space(s):
+    return s.strip() if isinstance(s, str) else s

authen/util/xlsf.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+import re
+import datetime
+from decimal import Decimal
+
+import xlwt
+
+class CsvFile:
+    """
+    encode 文件编码
+    output 导出文件路径 *
+    """
+
+    def __init__(self, encoding='utf-8', output=None):
+        self.output = output
+
+    def init(self, record):
+        """
+        record list对象
+        """
+        with open(self.output, "w") as f:
+            for eachline in record:
+                f.write(",".join(eachline)+"\r\n")
+
+class XlsFile:
+    """
+    encode 文件编码
+    output 导出文件路径 *
+
+    """
+
+    def __init__(self, encoding='utf-8', output=None, style_kwargs={}):
+        self.workbook = xlwt.Workbook(encoding=encoding)
+        self.output = output
+        self.floatre = re.compile("^\d+\.\d{2}$")
+
+        self.style_int = xlwt.XFStyle()
+        self.style_int.num_format_str = '0'
+
+        self.style_float = xlwt.XFStyle()
+        self.style_float.num_format_str = style_kwargs.get("style_float", '0.00')
+
+        self.style_normal = xlwt.XFStyle()
+        self.style_normal.num_format_str = '@'
+
+        self.style_date = xlwt.XFStyle()
+        self.style_date.num_format_str = style_kwargs.get("style_date", 'YYYY-MM-DD')
+
+    def init(self, record):
+        """
+        给一个字典, key是工作区, value是内容, value为list
+        """
+        for sheet, data in record.items():
+            worksheet = self.workbook.add_sheet(sheet)
+            for row_index, item in enumerate(data):
+                for column_index, v in enumerate(item):
+                    if isinstance(v, Decimal) or isinstance(v, float):
+                        worksheet.write(row_index, column_index, 
+                            label=float(v), style=self.style_float)
+                    elif isinstance(v, int) or isinstance(v, long):
+                        worksheet.write(row_index, column_index, 
+                            label=int(v), style=self.style_int)
+                    elif isinstance(v, datetime.date):
+                        worksheet.write(row_index, column_index, 
+                            label=v, style=self.style_date)
+                    else:
+                        worksheet.write(row_index, column_index, label=str(v))
+        self.workbook.save(self.output)

setup.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+
+setup(
+    name='authen.util',
+    version=0.17,
+    url='http://git.aoyanming.com/authen/util.git',
+    license='GPL',
+    author='authen',
+    author_email='295002887@qq.com',
+    description='util func',
+    long_description=__doc__,
+    packages=find_packages(exclude=['ez_setup']),
+    namespace_packages=['authen'],
+    include_package_data=True,
+    install_requires=[
+        'setuptools'
+    ],
+    classifiers=[
+        "Framework :: Plone",
+        "Framework :: Zope2",
+        "Framework :: Zope3",
+        "Programming Language :: Python",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    entry_points="""
+    # -*- Entry points: -*-
+    """
+)