app/activities/TransactionEditor.py

@@ -386,17 +386,17 @@ class TransactionEditor(FocusWidget):
         self.components.update({
             'bottom_pane': Columns([
                 Pile([
-                    _widgets[x] for x in ['description', 'dbview']
+                    _widgets[x] for x in [ 'description', 'dbview', ]
                 ]),
                 (self.graph.total_width+2, Pile([
                     LineBox(
                         AttrMap(self.components['badge'], 'badge'),
                         title="Current Price", title_align='left',
                     ),
-                    LineBox(
-                        self.graph,
-                        title="Historic Price", title_align='left'
-                    ),
+                    #LineBox(
+                    #    self.graph,
+                    #    title="Historic Price", title_align='left'
+                    #),
                 ])),
             ])
         })

app/data/filter.py

@@ -4,7 +4,6 @@
 # All rights reserved
 #
 # THIS SOFTWARE IS PROVIDED AS IS WITHOUT WARRANTY
-from bottle import FormsDict
 from typing import Iterable, List, Tuple, Union
 
 
@@ -35,7 +34,7 @@ def get_query_param(include, exclude):
 
 
 def get_filter(
-    query: FormsDict, allow: Iterable[str] = None
+    query: dict, allow: Iterable[str] = None
 ) -> dict[str, Tuple[List[str], List[str]]]:
     return {
         k: get_include_exclude(

blacklist.txt

@@ -0,0 +1,23 @@
+Kroger
+Simple Truth Organic
+Organic
+Huggies
+Ghirardelli
+Fresh
+Premium
+The Kitchen
+Gluten Free
+The Odd Bunch
+pdm
+Silk baby
+99% water & plastic free
+Woolworths
+Fresh fruit
+Yellow loose
+Green loose
+Fresh vegetable
+per kg
+loose
+kg prepacked
+minimum 5 per pack
+bulb single

common.gawk

@@ -0,0 +1,226 @@
+{
+    pattern_weight = "(oz|lb|lbs|g|kg)"
+    pattern_volume = "(ml|l|L|ML|fl oz|fl. oz|fl.oz)"
+    pattern_count  = "(each|piece|pieces|bag|bags|sheet|sheets|count|ct|pack|pk)"
+    unit_any = "(" pattern_weight "|" pattern_volume "|" pattern_count ")"
+    ""
+}
+
+function canonical_unit(u,    lu) {
+    lu = tolower(u)
+
+    # Weight
+    if (lu ~ /^lb$|^lbs$/) return "lb"
+    if (lu == "oz")        return "oz"
+    if (lu == "g")         return "g"
+    if (lu == "kg")        return "kg"
+
+    # Volume
+    if (lu == "ml")        return "mL"
+    if (lu == "l")         return "L"
+    if (lu ~ /^floz$|^fl oz$|^fl\.oz$|^fl\. oz$|^fl\. oz\.$/) return "fl. oz."
+
+    # Count
+    if (lu ~ /^each$|^piece$|^pieces$/) return "Pieces"
+    if (lu ~ /^count$|^ct$/)            return "Pieces"
+    if (lu ~ /^pack$|^pk$/)             return "Pieces"
+    if (lu ~ /^bag$|^bags$/)            return "Bags"
+    if (lu ~ /^sheet$|^sheets$/)        return "Sheets"
+
+    return u
+}
+
+function load_products(filename, line) {
+    n_products = 0
+    while ((getline line < filename) > 0) {
+        products[++n_products] = line
+    }
+    close(filename)
+}
+
+function load_translations(filename, line,  i, j) {
+    n_translate = 0
+    while ((getline line < "translate.txt") > 0) {
+        # Parse format: source=>target
+        if (match(line, /(.*)=>\s*(.*)/, arr)) {
+            source[++n_translate] = arr[1]
+            target[n_translate] = arr[2]
+        }
+    }
+    close("translate.txt")
+
+    # Sort by length of source descending (longest first)
+    for (i = 1; i <= n_translate-1; i++) {
+        for (j = i+1; j <= n_translate; j++) {
+            if (length(source[i]) < length(source[j])) {
+                tmp = source[i]; source[i] = source[j]; source[j] = tmp
+                tmp = target[i]; target[i] = target[j]; target[j] = tmp
+            }
+        }
+    }
+}
+
+function translate_string(s,    i) {
+    for (i = 1; i <= n_translate; i++) {
+        gsub(source[i], target[i], s)  # exact substring match, case-insensitive if IGNORECASE=1
+    }
+    return s
+}
+
+function load_blacklist(filename, line, i, j) {
+    n_blacklist = 0
+    while ((getline line < "blacklist.txt") > 0) {
+        blacklist[++n_blacklist] = line
+    }
+    close("blacklist.txt")
+
+    # Sort blacklist by length descending (longest first)
+    for (i = 1; i <= n_blacklist-1; i++) {
+        for (j = i+1; j <= n_blacklist; j++) {
+            if (length(blacklist[i]) < length(blacklist[j])) {
+                tmp = blacklist[i]; blacklist[i] = blacklist[j]; blacklist[j] = tmp
+            }
+        }
+    }
+}
+
+function strip_blacklist(s,    i) {
+    for (i = 1; i <= n_blacklist; i++) {
+        gsub(blacklist[i], "", s)   # exact substring match
+    }
+    # collapse multiple spaces
+    gsub(/[[:space:]]+/, " ", s)
+    sub(/^ /, "", s)
+    sub(/ $/, "", s)
+    return s
+}
+
+function parse_organic(line){
+    return (tolower(line) ~ / org | org\.|organic/) ? "true" : "false"
+}
+
+function parse_total(line,  price_paid, m){
+    if (match(line, /\$[0-9]+\.[0-9][0-9]/, m)) {
+        price_paid = m[0]
+        gsub(/^\$/, "", price_paid)
+    } else {
+        return ""
+    }
+    return price_paid
+}
+
+function parse_amount_unit(line, description,    d, q1, q2, q3, rest, dunit, damount, patA, patB){
+    # Pattern A: 2.42 lbs x $1.99 each
+    patA = "([0-9.]+)[ ]*(" unit_any ")[ ]*x"
+    patB = "[^$]([0-9.]+)[ ]*(" unit_any ")[ ]*"
+    patC = "^([0-9.]+)[ ]*x"
+    if (match(line, patA, q1)) {
+        amount = q1[1]
+        unit   = canonical_unit(q1[2])
+    }
+    # Pattern B: 6ea/0.850 kg
+    else if (match(line, patB, q2)){
+        amount = q2[1]
+        unit   = canonical_unit(q2[2])
+    }
+    # Pattern C: 1 x $9.99 each
+    else if (match(line, patC, q3)) {
+        amount = q3[1]
+        # unit resolved later
+    }
+    rest = description
+    dunit = ""
+    damount = ""
+    while(match(rest, "([0-9.]+)[ ]*(" unit_any ")", d)){
+        rest = substr(rest, RSTART + RLENGTH)
+        dunit = canonical_unit(d[2])
+        damount = d[1]
+    }
+    if((dunit != "" && damount != "" && damount + 0 > amount + 0) && (unit == "Pieces" || unit == "")) {
+        unit = dunit
+        amount = damount
+    }
+
+    if (amount == "") amount = 1
+    if (unit == "")   unit = "Pieces"
+    return ""
+}
+
+###############################################################################
+# Compute Levenshtein distance (standard implementation)
+###############################################################################
+function levenshtein(a, b,    la, lb, i, j, cost, d) {
+    la = length(a)
+    lb = length(b)
+
+    # Create matrix
+    for (i = 0; i <= la; i++) d[i,0] = i
+    for (j = 0; j <= lb; j++) d[0,j] = j
+
+    # Fill dynamic table
+    for (i = 1; i <= la; i++) {
+        for (j = 1; j <= lb; j++) {
+            cost = (substr(a,i,1) == substr(b,j,1) ? 0 : 1)
+            d[i,j] = d[i-1,j] + 1       # deletion
+            if ((tmp = d[i,j-1] + 1) < d[i,j]) d[i,j] = tmp  # insertion
+            if ((tmp = d[i-1,j-1] + cost) < d[i,j]) d[i,j] = tmp  # substitution
+        }
+    }
+    return d[la,lb]
+}
+
+function norm(s) {
+    s = tolower(s)
+    gsub(/[^a-z0-9 ]/, "", s)
+    gsub(/[ ]+/, " ", s)
+    sub(/^ /, "", s)
+    sub(/ $/, "", s)
+    return s
+}
+
+# returns 1 if at least one word in common, 0 otherwise
+function has_shared_word(line, prod,    dist, desc, dw, pw, i, j) {
+    desc = norm(line)
+    prod = norm(prod)
+
+    n_desc = split(desc, dw, " ")
+    n_prod = split(prod, pw, " ")
+
+    for (i = 1; i <= n_desc; i++){
+        if (length(dw[i]) < 3) continue
+        for (j = 1; j <= n_prod; j++){
+            dist = levenshtein(dw[i], pw[j]) 
+            dist = 1 - (dist / (length(dw[i]) > length(pw[j]) ? length(dw[i]) : length(pw[j])))
+            if (dw[i] == pw[j] || dist > 0.8) return 1
+        }
+    }
+
+    return 0
+}
+
+function fuzzy_product(description,    best, bestdist, dprod, descnorm, pnorm, dist, i) {
+    descnorm = norm(description)
+    bestdist = 0
+    best = ""
+
+    for (i = 1; i <= n_products; i++) {
+        pnorm = norm(products[i])
+        if (!has_shared_word(descnorm, pnorm)) continue
+        dist = levenshtein(descnorm, pnorm)
+        if (index(descnorm, pnorm) > 0) dist -= 1 
+        #print descnorm " " pnorm " " dist
+        dist = 1 - (dist / (length(descnorm) > length(pnorm) ? length(descnorm) : length(pnorm)))
+
+        if (dist > bestdist) {
+            bestdist = dist
+            best = products[i]
+        }
+    }
+
+    #print description " " best " " bestdist 
+    if (bestdist >= 0.25)
+        return best
+    else
+        return ""
+}
+

parse-receipt.gawk

@@ -0,0 +1,73 @@
+#!/usr/bin/gawk -f
+
+# Usage:
+#   gawk -v date="2025-11-02 11:07" -v store=Kroger -f parse-receipt.gawk receipt.txt
+
+
+@include "common.gawk"
+BEGIN {
+    IGNORECASE = -1
+    load_products("products.txt")
+    load_blacklist("blacklist.txt")
+    load_translations("translate.txt")
+
+    if (store == "Kroger"){
+        RS = ""      # blank-line records
+        FS = "\n"    # fields separated by newlines
+    }
+    else {
+        FS = "\t"    # fields separated by tab
+    }
+}
+####################################################################
+# Main processing
+####################################################################
+{
+    if (store == "Kroger"){
+        split($1, p, /\t+/)
+        desc = p[1]
+
+        price = parse_total($1)
+        if (price == "") next
+        
+        organic = parse_organic(desc)
+
+        amount = ""
+        unit = ""
+        parse_amount_unit($2, desc)
+
+    }
+    else {
+        if (NF == 1) next
+        desc = $1
+
+        if (store == "Countdown"){
+            price_field = $5
+        }
+        else {
+            price_field = $4
+        }
+        price = parse_total(price_field)
+        if (price == "") next
+        
+        organic = parse_organic(desc)
+
+        amount = ""
+        unit = ""
+        if (store == "Countdown"){
+            amount_unit_field = $3
+        }
+        else {
+            amount_unit_field = $2 " " $3
+        }
+
+        parse_amount_unit(amount_unit_field, desc)
+
+    }
+    cleaned = translate_string(strip_blacklist(desc))
+
+    product = fuzzy_product(cleaned)
+
+    printf "CALL insert_transaction('%s', $store$%s$store$, $descr$%s$descr$, %s, $unit$%s$unit$, %s, $produ$%s$produ$, %s);\n",
+        date, store, desc, amount, unit, price, product, organic
+}

parse-receipt.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#set -x
+
+FNAME="${0##*/}"
+
+usage(){
+    cat <<EOF
+  Usage:
+    ./"${0##*/}" date="2025-11-02 11:07" store="Kroger" < receipt.txt
+EOF
+}
+
+vars=()
+for param in "$@"
+do
+    if [ "$param" == "-h" ]; then usage; exit 1; fi
+
+    vars+=(-v "$param")
+done
+
+<&0 gawk "${vars[@]}" -f "${FNAME/.sh/.gawk}"

products.txt

@@ -0,0 +1,447 @@
+A2 Cream
+Active Yeast
+Adzuki Beans
+Alberts Eggs
+Almond Butter
+Almond Milk
+Almonds
+Apple Cider Vinegar
+Apple Juice
+Apple Sauce
+Apples
+Apricots
+Arrowroot Powder
+Artisan Bread
+Asparagus
+Avocado
+BEE Lemonade
+BRB IPA
+Baby Food
+Baguette
+Baking Soda
+Bananas
+Bandaids
+Barn Eggs
+Bars
+Beans Canned
+Beef Chuck
+Beef Jerky
+Beef Marrow Bones
+Beef Mince
+Beef Mince 10% Fat
+Beef Mince 13% Fat
+Beef Mince 15% Fat
+Beef Mince 18% Fat
+Beef Patties
+Beef Rib Roast
+Beef Roast
+Beef Rump
+Beef Sausages
+Beef Scotch Fillet
+Beef Shin
+Beef Stir Fry
+Beef Stock
+Beef Topside
+Beetroot
+Beetroot Canned
+Blade Steak
+Blueberries Frozen
+Bok Choy
+Bolar Roast
+Bone Broth Powder
+Boneface Hazy IPA
+Borlotti Beans
+Brazil Nuts
+Breakfast Cereal
+Brewers Yeast
+Brie Cheese
+Brisket
+Broccoli
+Broccoli Cauli Mix Frozen
+Brown Lentils
+Brown Rice
+Brown Sugar
+Buckwheat Groats
+Buns
+Burger Patties
+Butter
+Butternut Squash
+Cabbage
+Cabernet Sauvignon
+Cacao Powder
+Cage Free Eggs
+Candied Ginger
+Canned Apricots
+Canned Corn
+Canned Peaches
+Canned Salmon
+Canned Tomatoes
+Canned Tuna
+Capsicum
+Cardamon
+Carrots
+Cashews
+Cat Food Naturals
+Cat Food Purina
+Cauliflower
+Cauliflower Rice
+Cayenne Powder
+Celery
+Cheddar Cheese
+Cheese Sticks
+Cherries Fresh
+Chewy Bars
+Chia Seeds
+Chicken Drumsticks
+Chicken Frames
+Chicken Kebabs
+Chicken Livers
+Chicken Mince
+Chicken Nibbles
+Chicken Sausages
+Chicken Shredded
+Chicken Stir Fry
+Chicken Tender
+Chicken Thighs
+Chicken Thighs BL
+Chickpeas
+Chickpeas Canned
+Chilled Juice
+Chilli Powder
+Choc Covered Almonds
+Choc Covered Peanuts
+Chocolate Chips
+Chocolate Covered Cherries
+Chopped Pecans
+Cinnamon, Ground
+Cinnamon, Stick
+Coconut Cream
+Coconut Desiccated
+Coconut Milk
+Coconut Oil
+Coconut Water
+Coconut Yoghurt
+Coffee Beans
+Coffee Grounds
+Colby Cheese
+Collagen
+Concentrated Juice
+Cookie Pack
+Cookie Time
+Corn Chips
+Corn Cobb
+Corned Beef
+Cornflour
+Cos Lettuce
+Cottage Cheese
+Cotton Tips
+Courgettes
+Crackers
+Cranberry Juice
+Cream
+Cream Cheese
+Crispy Mix
+Cruskits
+Cucumber
+Currant Juice
+Curry Paste
+Dark Chocolate
+Demerara Sugar
+Dish Soap
+Donuts
+Doritos
+Double Cream
+Double Vision Beer
+Dried Apricots
+Dried Bananas
+Dried Dates
+Dried Figs
+Dried Mango
+Dried Pineapple
+Dried Strawberries
+Dry Peas
+Edam Cheese
+Eggplant
+Epsom Salt
+Export Gold
+FR Chicken Breast
+FR Chicken Drumsticks
+FR Chicken Mince
+FR Chicken Thighs
+FR Chicken Thighs BL
+FR Chicken Whole
+Fancy Lettuce
+Fenugreek Seeds
+Feta
+Fire lighters
+Fish Fillets
+Fish Frames
+Fish Heads
+Fish Sauce
+Fizzy Hairy Lemon
+Fizzy Tablets
+Flax Seeds
+Founders Beer
+Free Range Eggs
+French Lentils
+Frozen Cherries
+Frozen Green Beans
+Frozen Mango
+Frozen Meal
+Frozen Peas
+Frozen Pizza
+Frozen Pop
+Fruit Bars
+Fruit Strips
+Gai Lan
+Garabage Bags
+Garam Masala
+Garlic
+Garlic Bread
+Garlic Powder
+Gelatin
+Ginger
+Ginger Beer
+Ginger Ground
+Good George Cider
+Granola
+Grapefruit Juice
+Grapes
+Ground Almonds
+Ground Cumin
+Guacamole
+Haagen Lager
+Hemp Oil
+Hemp Protein
+Hemp Seeds
+Herbal Tea
+Honey
+Hot Chicken
+Hot Chilli
+Hot Chocolate
+Hot Dog
+Hot Dog Rolls
+Hummus
+Ice Cream
+Ice Cream Cone
+Immunity Juice
+Jalapenos Canned
+Jam
+Jar of Chillis
+Kaffir Lime Leaves
+Kassori Methi
+Kefir Culture
+Kelp
+Kelp Salt
+Kidney Beans Canned
+Kiwifruit
+Kiwifruit Dried
+Kombucha
+Kumara
+Lamb Bones
+Lamb Cubes
+Lamb Leg
+Lamb Mince
+Lamb Mince 13% Fat
+Lamb Neck Chops
+Lamb Pieces
+Lamb Riblets
+Lamb Roast
+Lamb Rump
+Lamb Shank
+Lamb Shoulder
+Laundry Detergent
+Lemon Fresh
+Lettuce
+Linssed
+Low Sugar Soda
+M&Ms Mini
+MOA Hazy
+MOA IPA
+Macs Beer
+Macs IPA
+Madtree Beer
+Magnesium Fizz
+Mandarins
+Mango
+Maple Syrup
+Marmite
+Marshmallows
+Mayo
+Meatballs
+Merlot
+Microgreens
+Mild Cheese
+Milk Chocolate
+Milk Standard
+Misc Lollies
+Miso
+Mixed Fruit Frozen
+Mixed Nuts
+Mixed Red Wine
+Mixed Spice
+Molasses
+Monkfruit
+Monteiths Hazy Pale Ale
+Mozzarella Cheese
+Mung Bean Sprouts
+Mung Beans
+Mushroom Flat
+Mushroom Portabello
+Mushrooms
+Mustard Powder
+Mustard Seeds
+Mutton Chops
+Nappies Size 1
+Nappies Size 3
+Nappies Size 4
+Neck Chops
+Nut Bars
+Nutritional Yeast
+OSM bars
+Oat Bars
+Olive Oil
+Olive Oil Light
+Olives Bag
+Olives Jar
+Onions
+Onions Red
+Orange Juice
+Oranges
+Oreo Cookies
+Organic Cheese
+Organic Chicken
+Paleo Cake
+Paleo Protein
+Paprika
+Parmesan Cheese
+Parrotdog Birdseye Hazy IPA
+Parsnips
+Pasta
+Pasta Sauce
+Peaches
+Peanutbutter
+Peanuts
+Pears
+Persimmons
+Pineapple Canned
+Pineapple Fresh
+Pinenuts
+Pinot Noir
+Pita
+Pizza Bases
+Plums
+Potato Crisps
+Potatoes
+Powdered Milk
+Prepared Pies
+Prepared Pizza
+Prunes
+Psyllium Husk
+Puff Pastry
+Puffed Rice
+Pukka Tea
+Pumpkin Seeds
+Pumpkin Squash
+Quinoa
+Raisins
+Raspberries
+Raw Milk
+Red Blend
+Red Lentils
+Rhinegeist Beer
+Rice Cakes
+Rice Crackers
+Rice Noodles
+Roasted Tomatoes
+Rochdale Cider
+Rolled Oats
+Rooibos Tea
+Rye Flour
+Ryecorn
+Salad Bag
+Salad Mix
+Salami
+Salmon Steaks
+Salsa
+Salt
+Sardines
+Saurkraut
+Seaweed
+Shiraz
+Short Pastry
+Sirloin Steak
+Skirt Steak
+Sliced Cheese
+Smoked Salmon
+Snack Balls
+Snackachangi Chips
+Soda
+Soda Syrup
+Soup Pumpkin
+Soup Refrigerated
+Soup Tomato
+Sour Cream
+Soy Sauce
+Spinach
+Squash
+Starter
+Stoke Beer Bright IPA
+Stoke Beer Hazy Pale Ale
+Stoke Beer Pilsner
+Strawberries
+Strawberries Frozen
+Strawberry Seconds
+String Cheese
+Subscription Box
+Sugar Free Chocolate
+Sugar, Coconut
+Sultanas
+Sunflower Oil
+Sunflower Seeds
+TNCC Lollies
+Tangelo
+Tasty Cheese
+Tempeh
+Tissues
+Toilet Paper
+Tomato Fresh
+Tomato Juice
+Tomato Paste
+Tomato Sauce
+Tonic Water
+Tortillas
+Tri Tip Beef
+Tulsi Tea
+Tumeric
+Tumeric Milk
+Tuna Steaks
+Turkey Bacon
+Vanilla
+Vegetable Bouillon
+Vegetable Juice
+Veggie Crisps
+Veggie Mix
+Vinegar
+Vintage Cheese
+Vitamin Water
+Voodoo Ranger Beer
+Walnut
+Walnuts
+Water
+Watermelon
+Wheat Berries
+Wheat Bread
+Whey Protein
+White Flour
+White Rice
+White Sugar
+Whittakers Chocolate
+Whole Chicken
+Whole Fish
+Whole Oats
+Wholemeal Flour
+Wipes
+Xcut Blade Steak
+Xcut Short Rib
+Yoghurt Dairy

reconcile.py

@@ -54,6 +54,7 @@ STORE_CODES = {
     'WHATAWHATA BERRY FARM': 'Farm',
     'TAUPIRI DAIRY': 'TD',
     'Dreamview Cre': 'DV',
+    'PMT TO FC12-3189-0013895-00': 'DV',
     'new world': 'NW',
     'BIN INN': 'BI',
     'MAPBI': 'BI',
@@ -75,6 +76,10 @@ STORE_CODES = {
     'WWNZ' : 'CD',
     'Farmers Market': 'FM',
     'MAPMISC': 'MISC',
+    'Four Square': 'FS',
+    'KROGER': 'KROGE',
+    'MITRE 10': 'M10',
+    'PACIFIC HEALTH': 'PH',
 }
 
 conn = psycopg.connect(f"{host} dbname=grocery {user} {password}")
@@ -84,8 +89,9 @@ output = []
 def get_record_from_database(date, store, tags):
     cur.execute(get_statement(date, store))
     #print(cur.mogrify(get_statement(date, store)))
+    rows = [ i for i in cursor_as_dict(cur) ]
     return sum([
-        row['price'] for row in cursor_as_dict(cur) if not (
+        row['price'] for row in rows if not (
             tags & set(row['tags'] or [])
         )
     ])

translate.txt

@@ -0,0 +1,4 @@
+diapers=>nappies
+sweet potato=>kumara
+sweet potatoes=>kumaras
+yogurt=>yoghurt