Difference between revisions of "Python"

From Torben's Wiki
m (File Access)
m (ReadConfigFile to connect to PostgreSQL)
Line 531: Line 531:
  
 
====ReadConfigFile to connect to PostgreSQL====
 
====ReadConfigFile to connect to PostgreSQL====
from {http://www.postgresqltutorial.com/postgresql-python/connect/}
+
from [http://www.postgresqltutorial.com/postgresql-python/connect/]
 
database.ini
 
database.ini
 
  [postgresql]
 
  [postgresql]

Revision as of 17:42, 10 August 2019


Template

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# see https://google.github.io/styleguide/pyguide.html

import os
import os.path  # os.path - The key to File I/O
import argparse
import configparser
import logging


# Init Logging
# from [1]
# create logger
logger = logging.getLogger('myLogger')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
logger_fh = logging.FileHandler('log.log')
logger_fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
logger_ch = logging.StreamHandler()
logger_ch.setLevel(logging.INFO)
# create formatter and add it to the handlers
# %(name)s = LoggerName, %(threadName)s = TreadName
logger_formatter = logging.Formatter(
    '%(asctime)s - %(levelname)s - %(message)s')
logger_fh.setFormatter(logger_formatter)
logger_ch.setFormatter(logger_formatter)
# add the handlers to the logger
logger.addHandler(logger_fh)
logger.addHandler(logger_ch)
#
logger.debug('DebugMe')
logger.info('Starting')
logger.warning('Attention')
logger.error('Something went wrong')
logger.critical('Something seriously went wrong')


# Read Commandline Parameters
# construct the argument parser and parse the arguments
arg_parser = argparse.ArgumentParser()
# -h comes automatically
# Boolean Parameter
arg_parser.add_argument("-v", "--verbose", help="increase output verbosity",
                        action="store_true")  # store_true -> Boolean Value
#
args = vars(arg_parser.parse_args())
if args["verbose"]:
    logger_ch.setLevel(logging.DEBUG)


# Read config file
config = configparser.ConfigParser()
config.read('config.ini')
# print(config.getfloat('Section1', 'Value1'))


# Dummy variables
s = "asdf"
l = ["a", "b", "c"]
d = {'keyx': 'valuex', 'keyy': 'valuey'}
d['keyz'] = 'valuez'


# File access
fileOut = "out/1/out.txt"
(filepath, fileName) = os.path.split(fileOut)
(fileBaseName, fileExtension) = os.path.splitext(fileName)
os.makedirs(filepath, exist_ok=True)

FILE = open(fileOut, "w")  # w = overWrite file ; a = append to file
FILE.write(s + "\n")
FILE.close()

fileIn = fileOut
FILE = open(fileIn, "r")
cont_list = (FILE.read()).split("\n")
FILE.close()

Object Oriented Template

class myDevice():
    def __init__(self, devicename="", verbose=False):
        # name of the device (e.g. for log messages)
        self.devicename = devicename
        self.verbose = verbose   # whether to log information or be quiet
    def log(self, msg):
        print(msg)
class SMU236(myDevice):
    def __init__(self, gpibaddress, devicename="", verbose=False):
        myDevice.__init__(self, devicename, verbose)
        self.gpibaddress = gpibaddress
if __name__ == "__main__":
    SMU = SMU236(1234)
    SMU.log("Starting")

method using self.x as default parameter

    def takeScreenshot(self, x=None):
        if x is None:
            x = self.windowsGeo['x']

Basics

Naming Conventions

Google Python Style Guide:

module_name, package_name, ClassName, method_name, ExceptionName, function_name, GLOBAL_CONSTANT_NAME, global_var_name, instance_var_name, function_parameter_name, local_var_name

See Category:Python for more stuff

# delete a variable
del var
# check if variable is defined
if xyz in locals() :
# for object oriented projects:
if "xyz" in self.__dict__.keys(): 

sleep for a while

import time
time.sleep(60)

wait for user input

raw_input("Press Enter to close")


Strings

# num <-> str
s= str (i) # int to string
f = float("1.234") # str -> float
str(round(f, 1)) # round first 

text = text.strip() # trim spaces
print (text.lower()) # lower / upper cases
print("Good Morning!", end = ) # print without linebreak

text = prompt("Enter Text: ") # get string from prompt

print "Renner =", i
# r in front of string disables "\" masking 
dest_path = r"\\server\dir1\file.txt"
# handling substrings
a = "abcd"
b = a[:1] + "o" + a[2:] 
> 'aocd'

# find a substring:
x in s
> True / False

s * 5
>>> s+s+s+s+s

s.replace(x, y)

# convert int i to string with leading zeros if i<100
print "%03d" % (i)

# place formatted numbers in a string / sprintf
"The %03i %s cost %f euros" % (3, "beers", 11.50)
> 'The 3 beers cost 11.500000 euros'

"The length is %.2f meters" % 72.8958
>'The length is 72.90 meters'

p= "%.1f%%/min" % precent

Find a substring
myString="Hello there !bob@"
mySubString=myString[myString.find("!")+1:myString.find("@")]

the string lib

import string
string.capwords(s) # upper case first letter of each word and also removes multiple and trailing spaces

Lists

like arrays in Perl

L = [1,2,3,4,5,6]
L = [x for x in range(10)]
L = s.split () # split string s by spaces, use (s.split(",")) to split on "," etc
len(L)
L[0:10] # get elements 0-10
for a in L:
  ...

M = L.copy # clone L
M = L[:] # clone L
M = L # M is a link to L
L.append(x) # append a single element
L.extend(M) # put elements of list M to the end of List L
L.insert(i, x) # insert item x at position int i
L.pop() # returns and removes the last item
L.pop(i) # returns and removes the item at position int i
L.reverse()
L.sort()
L.remove(x) # removes the first occurrence of item x
L.count(x) # how many items x are in the list
L.index(x) # gives the position of the first x in list
s="".join(L)
x in L 
x not in L

# List to string
"\n".join(L)

# modify item in list
for idx, line in enumerate(cont):
  if "K1001/1" in line:
    line = "K1001/1 Test Nr " + str(i) + "\n"
    cont[idx] = line
    break

Tupels

Ordered sequence, with no ability to replace or delete items

L = (1,2,3,4,5,6)

Dictionaries

like hash in Perl

d = {'keyx': valuex, 'keyy': valuey}
d['keyz'] = valuez
tel.keys()
['keyx', 'keyy', 'keyz']
del d['keyy']

len(d)
d.clear()
d.copy()
d.keys()
d.values()
d.items() # returns a list of tuples (key, value)
d.get(k) # returns value of key k
d.get(k, x) # returns value of key k; if k is not in d it returns x
d.pop(k) # returns and removes item k
d.pop(k, x) # returns and removes item k; if k is not in d it returns x
x in d
x not in d

sort keys

for userid in sorted (dict.keys()):

MultiDim Dictionaries

dicProductivity = {} 
dicProductivity['Cursor'] = {}
dicProductivity['Cursor']['Nr'] = 1
dicProductivity['Cursor']['Prod'] = 1.909E18
dicProductivity['Cursor']['Cost'] = 0
dicProductivity['Cursor']['Img'] = 'templates/Shop01Cursor.png'
dicProductivity['Grandma'] = {}
dicProductivity['Grandma']['Nr'] = 2
dicProductivity['Grandma']['Prod'] = 1.725E18
dicProductivity['Grandma']['Cost'] = 0
dicProductivity['Grandma']['Img'] = 'templates/Shop02Grandma.png'

for k in dicProductivity.keys() :
   print(k)
   if 'Img' in dicProductivity[k] :
       print("ja")

Loops

while i <= 100:
  ...
  if sth:
    break

for i in range(1, 5):
  print i
  if sth:
    continue
else:
  print 'The for loop is over'

for f in files :

inline if (requires a dummy else):

print("something") if self.verbose else 0

asserts function argument validation

aus Python Kurs von Carsten Knoll

def eine_funktion(satz, ganzzahl, zahl2, liste):
  if not type(satz) == str:
    print "Datentpyfehler: satz"
    return -1
  if not isinstance(ganzzahl, int):
    print "Datentpyfehler: ganzzahl"
    return -2
  if not isinstance(liste, (tuple, list)):
    print "Datentpyfehler: liste"
    return -3
  # Kompakteste Variante (empfohlen): 
  assert zahl2 > 0, "Error: zahl2 ist nicht > 0" # Assertation-Error bei Nichterfuellung
def F(x):
  if not isinstance(x, (float, int)):
    msg = "Zahl erwartet, %s bekommen" % type(x)
    raise ValueError(msg)
  return x**2

better:

def F(x):
  assert isinstance(x, (float, int)), "Error: x is not of type float or int"
  return x**2
assert variant in ['normal', 'gray', 'cannyedge'], "Error: variant is not in 'normal', 'gray', 'cannyedge'"

Imports

import sys
import datetime
import time
import math
import random
import os.path
# Import my files
import MyFile # without tailing .py
# import a file, not stored in the same folder
import sys
sys.path.append("../libs/MyFile ")

Math

see Python - Math for linear regression

get rid of the annoying integer division: [2]

from __future__ import division

Modulo

15 % 4
--> 3

Random

import random
random.randint(1000000, 9999999)

Date and Time

http://pleac.sourceforge.net/pleac_python/datesandtimes.html

import datetime
d = datetime.date.today().strftime("%y%m%d")
d = datetime.datetime.today().strftime("%y%m%d-%H%M")

measure time elapsed

import time
timestart = time.time()
...
print(time.time() - timestart)

calculate time

import time
duration = 1234 # sec
print "ETA =",time.ctime(time.time()+duration)
array = time.localtime(time.time()+duration)

parsing iso dates

from datetime import datetime
date = datetime.fromisoformat('2017-01-01T12:30:59.000000')
date = datetime.fromisoformat(str[:-1]) # to remove "Z" from end

File Access

http://www.penzilla.net/tutorials/python/fileio/

Get filename of python script

from sys import argv
myFilename = argv[0]

Read filename from commandline parameter

import sys
for filename in sys.argv:

Split path into folder, filename, ext

(dirName, fileName) = os.path.split(f)
(fileBaseName, fileExtension)=os.path.splitext(fileName)

Filename without extension

fileOut = os.path.splitext(fileIn)[0] + "-edit.jpg"

Cross platform paths

currentdir = os.curdir
mysubdir = os.path.join(currentdir, "mysubdir")

Run external program

import subprocess
process = subprocess.run(["sudo", "du", "--max-depth=1", mydir], capture_output=True, text=True)
print (process.stdout)

old, depricated way:

os.system( "gnuplot " + ausgabedatei )

Globbing of file names

import glob
glob.glob('html/[0-9]*.html')

[3] You can use glob:

import glob, os
os.chdir("/mydir")
for file in glob.glob("*.txt"):
    print(file)

or simply os.listdir:

import os
for file in os.listdir("/mydir"):
    if file.endswith(".txt"):
        print(os.path.join("/mydir", file))

or if you want to traverse directory, use os.walk:

import os
for root, dirs, files in os.walk("/mydir"):
    for file in files:
        if file.endswith(".txt"):
             print(os.path.join(root, file))

Read file

Check if file / dir exists

import os.path  # os.path - The key to File I/O
os.path.exists("text.txt")
os.path.isfile("text.txt")
os.path.isdir("text")
os.path.isabs("/home/torben/text.txt") # Is it an absolute path
fh = open(filename,"r")
cont = fh.read()
# or
list = (fh.readlines())
# or
line =  fh.readline()
# or
for line in fh:
   print(line)
fh.close()

Write to File

fileOut= "out/1/out.txt"
(filepath, fileName) = os.path.split(fileOut)
# (fileBaseName, fileExtension) = os.path.splitext(fileName)
os.makedirs(filepath, exist_ok=True) # = mkdir -p

fh = open(fileOut,"w") 
# w = overWrite file ; a = append to file
# If running Python in Windows, "\n" is automatically replaced by "\r\n". To prevent this use
# fh = open(fileOut, "w", newline="\n")
fh.writelines(list)
# or
for line in list:
  fh.write(line)
fh.close()

# Force update of filecontents without closing it
fh.flush()

Touch file

if os.path.exists(fname):
  os.utime(fname, None)
else:
  open(fname, 'w').close()

File Meta Data

Get file size

import os
int (os.path.getsize("moinsen.txt") )

Read Timestamp (last modified)

lasttime = os.path.getmtime(fname)

Copy File

shutil.copyfile(fileTemp,
                os.path.join(dest_path, fileOut))

Delete

Delete file

os.remove(filename)

Directories / Folders

make dirs

if not os.path.isdir(d) :
  os.mkdir(d) # normal
  os.makedirs(d) # recursively= with all parents

Delete folder+contents

import shutil
shutil.rmtree(d)

Get list of files (not dirs) in directory

dirname = "/path/to/some/dir"
listoffiles = [ f for f in os.listdir(dirname) if os.path.isfile(os.path.join(dirSource,f)) ]
listoffiles.sort()

same with filter on fileext

listoffiles = [ f for f in os.listdir(dirSource) if os.path.isfile(os.path.join(dirSource,f)) and f.lower()[-4:] == ".gpx" ]

Templates/Snippets

Commandline Parameters

import argparse
parser = argparse.ArgumentParser()  # construct the argument parser and parse the arguments
# -h comes automatically

# Boolean Parameter
parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")  # store_true -> Boolean Value

# Choice Parameter
# restrict to a list of possible values / choices
# parser.add_argument("--choice", type=int, choices=[0, 1, 2], help="Test choices")

# Positional Parameter (like text.py 123)
# parser.add_argument("num", type=int, help="Number of things")

# Required Parameter
# parser.add_argument("-i", "--input", type=str, required=True, help="Path of input file")

# Optional Parameter
parser.add_argument("-n", "--number", type=int, help="Number of clicks")
# Optional Parameter with Default
parser.add_argument("-s", "--seconds", type=int, default=secDefault, help="Duration of clicking, default = %i (sec)" % secDefault)

args = vars(parser.parse_args())

if args["verbose"]:
    pass # do nothing
   # print ("verbosity turned on") 
if args["number"]:
    print("num=%i" % args["number"])

ReadConfigFile

Config.ini

[Section1]
Cursor         = 205E18
Grandma        =  18E18
Farm           =  11E18
Mine           = 514E18
Factory        = 155E18

test.py

from configparser import ConfigParser
config = ConfigParser()
config.read('Config.ini')
print(config.getfloat('Section1', 'Cursor'))

ReadConfigFile to connect to PostgreSQL

from [4] database.ini

[postgresql]
host=dbhost
port=5432
database=dbname
user=dbuser
password=dbpass

config.py

from configparser import ConfigParser
def config(filename='database.ini', section='postgresql'):
   parser = ConfigParser()
   parser.read(filename)
   # get section, default to postgresql
   db = {}
   if parser.has_section(section):
       params = parser.items(section)
       for param in params:
           db[param[0]] = param[1]
   else:
       raise Exception(
           'Section {0} not found in the {1} file'.format(section, filename))
   return db

main.py

import psycopg2
from config import config 

def connect():
   """ Connect to the PostgreSQL database server """
   conn = None
   try:
       params = config() # read connection parameters
       print('Connecting to the PostgreSQL database...')
       conn = psycopg2.connect(**params)
       cur = conn.cursor() # create a cursor
       print('PostgreSQL database version:')
       cur.execute('SELECT version()')  # execute a statement
       db_version = cur.fetchone()
       print(db_version)
       cur.close()
   except (Exception, psycopg2.DatabaseError) as error:
       print(error)
   finally:
       if conn is not None:
           conn.close()
           print('Database connection closed.')

if __name__ == '__main__':
   connect()

Checking Operating System

import os
import sys
if os.name == 'posix':
  print 'posix/Unix/Linux'
elif os.name == 'nt':
  print 'windows'
else:
  print 'unknown os'
  sys.exit(1)

accessing os envrionment variables

import os
print(os.getenv('tmp'))

Logging

V2: File and STDOUT

import logging
# from [5]
# create logger
logger = logging.getLogger('myLogger')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler('mylog.log')
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')  # %(name)s = LoggerName, %(threadName)s = TreadName
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)  

logger.debug('DebugMe')
logger.info('Starting')
logger.warning('Attention')
logger.error('Something went wrong')
logger.critical('Something seriously went wrong')

V1: STDOUT

import logging
# Logging is nicer than print, as it can automatically add the threadname
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)s %(threadName)s: %(message)s',
                    )
logging.info('Starting')

Image/Picture/Photo Resize and Exif Modifying

from PIL import Image, ImageFilter  # pip install Pillow
import os, sys
fileIn = '2018-02-09 13.56.25.jpg'
# Read image
img = Image.open( fileIn )

Resize

# Resize keeping aspect ration -> img.thumbnail
# drops exif data, exif can be added from source file via exif= in save, see below
size = 1920, 1920
img.thumbnail(size, Image.ANTIALIAS)

Export file

fileOut = os.path.splitext(fileIn)[0] + "-edit.jpg"
try:
    img = Image.open(fileIn)
    img.save(fp=fileOut, format="JPEG", quality='keep')  # exif=dict_exif_bytes
    # JPEG Parameters
    # * qualitiy : 'keep' or 1 (worst) to 95 (best), default = 75. Values above 95 should be avoided.
    # * dpi : tuple of integers representing the pixel density, (x,y)
except IOError:
    print("cannot write file '%s'" % fileOut)

Export Progressive / web optimized JPEG

from PIL import ImageFile  # for MAXBLOCK for progressive export
fileOut = os.path.splitext(fileIn)[0] + "-progressive.jpg"
try:
    img.save(fp=fileOut, format="JPEG", quality=80, optimize=True, progressive=True)
except IOError:
    ImageFile.MAXBLOCK = img.size[0] * img.size[1]
    img.save(fp=fileOut, format="JPEG", quality=80, optimize=True, progressive=True)

Exif Data

import piexif  # pip install piexif
exif_dict = piexif.load(img.info['exif'])
print(exif_dict['GPS'][piexif.GPSIFD.GPSAltitude])
# returns list of 2 integers: value and donator  -> v / d
# (340000, 1000) => 340m
# (51, 2) => 25.5m

# Modify altitude
exif_dict['GPS'][piexif.GPSIFD.GPSAltitude] = (140, 1)  # 140m

# write to file
exif_bytes = piexif.dump(exif_dict)
fileOut = os.path.splitext(fileIn)[0] + "-modExif.jpg"
try:
    img.save(fp=fileOut, format="jpeg", exif=exif_bytes, quality='keep')
except IOError:
    print("cannot write file '%s'" % fileOut)

Template Matching

see Python - CV2

Optical Character Recognition (OCR)

see Python - OCR

Multi Threading

see Python - Multithreading

Send E-Mails

see Python - eMail

Regular Expressions

see Python - Regular Expressions

Hexadecimal

see Python - Hex

Compile to .exe

see Python - py2exe

Exceptions

Catch keyboard interrupt and do a "save exit"

try:
  FILE = open("out.txt","w")
  while 1:
    i+=1
    print i
except KeyboardInterrupt:
  FILE.close()

Catch all exceptions

try:
  [...]
except Exception, e:
  print "Exception raised: ", e

Custom Exceptions

try: 
  raise Exception('HiHo')

GUI Interactions

Take Screenshot

import pyautogui # (c:\Python\Scripts\)pip install pyautogui
# pyautogui does only support screenshots on monitor #1
...
screenshot = pyautogui.screenshot()
# screenshot = pyautogui.screenshot(region=(screenshotX,screenshotY, screenshotW, screenshotH))
screenshot = np.array(screenshot) 
# Convert RGB to BGR 
screenshot = screenshot[:, :, ::-1].copy()

Mouse Actions

def clickIt(x,y,key="") :
  x0, y0 = pyautogui.position()
  if key != "": # crtl, shift
    pyautogui.keyDown(key)
  pyautogui.moveTo(x, y, duration=0.2)
  pyautogui.click(x=x , y=y, button='left', clicks=1, interval=0.1)
  if key != "": # crtl, shift
    pyautogui.keyUp(key)
  pyautogui.moveTo(x0, y0)

Web Automation

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

# from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options

import os
import time
import glob

class StravaUserMapDL():
    def __init__(self):
        self.driver = webdriver.Firefox()

    def login(self):
        driver = self.driver
        url = "https://www.somewebpage.com"
        email = "myemail"
        password = "mypassword"
        driver.get(url)

        title = driver.title
        urlIs = driver.current_url
        cont = driver.page_source #  as string
        FILE = open(filename,"w") # w = overWrite file ; a = append to file
        FILE.write(cont)
        FILE.close()         

        # handle login if urlIs != url
        if (urlIs != url): 
            # activate checkbox 'remember_me'
            elem = driver.find_element_by_id('remember_me')
            if (elem.is_selected() == False):
                elem.click()
            assert elem.is_selected() == True
            elem = driver.find_element_by_id('email')
            elem.send_keys(email)
            elem = driver.find_element_by_id('password')
            elem.send_keys(password)
            elem.send_keys(Keys.RETURN)
            # Wait until login pages is replaced by real page
            urlIs = driver.current_url
            while (urlIs != url):
                time.sleep(1)
                urlIs = driver.current_url
            print (urlIs)

            # results = driver.find_elements_by_class_name('following')
            # results = driver.find_elements_by_tag_name('li')

            # print(results[0].text)
        assert (urlIs == url)


Unit Tests using Web Automation

import unittest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

#from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options

import os
import time

class PythonOrgSearch(unittest.TestCase):
#    def __init__(self,asdf):
#        self.driver = webdriver.Firefox() 

    def setUp(self):
        print ("setUp")
        # headless mode:
        # opts = Options()
        # opts.set_headless()
        # assert opts.headless  # Operating in headless mode
        # self.driver = webdriver.Firefox(options=opts)

        self.driver = webdriver.Firefox()

    def test_search_in_python_org(self):
        driver = self.driver
        driver.get("http://www.python.org")
        self.assertIn("Python", driver.title)
        elem = driver.find_element_by_name("q")
        elem.send_keys("pycon")
        elem.send_keys(Keys.RETURN)
        assert "No results found." not in driver.page_source
        print ("fertig: python_org")

    def tearDown(self):
        print ("tearDown")
        print ("close Firefox")
        self.driver.close() # close tab
        self.driver.quit() # quit browser
        # os._exit(1) # exit unittest without Exception


if __name__ == "__main__":
    try:
        unittest.main()
    except SystemExit as e:
        os._exit(1)

Debugging

Print name of the current function, useful to place at every function

import sys
...
if self.verbose:
  print("=== " + sys._getframe().f_code.co_name + " ===")
pip install ipython
...
from IPython import embed  
...
embed()  # to drop into iPython Shell from within the code

Or use the editor PyCharm or Visual Studio Code to set breakpoints to drop into python debugger