ynerant/nk20-scripts
1
0
Fork 0
mirror of https://gitlab.crans.org/bde/nk20-scripts synced 2025-10-30 22:59:52 +01:00
Code Issues Releases Wiki Activity

viva el encapsulation

Browse Source
This commit is contained in:
Pierre-antoine Comby
2020-05-25 01:18:24 +02:00
parent ddde4a2c53
commit df9888a31a
1 changed files with 6 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

513
management/commands/import_nk15.py
View File

@@ -1,458 +1,8 @@
#!/usr/env/bin python3 #!/usr/env/bin python3
import subprocess
import json
import datetime
import re
import pytz
import psycopg2 as pg
import psycopg2.extras as pge
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.core.management import call_command from django.core.management import call_command
from django.db import transaction
from django.core.exceptions import ValidationError
from django.utils.timezone import make_aware
from django.db import IntegrityError
from django.contrib.auth.models import User
from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction
from note.models import Note
from note.models import Alias
from note.models import (
TemplateCategory,
TransactionTemplate,
Transaction,
RecurrentTransaction,
MembershipTransaction,
SpecialTransaction,
)
from member.models import Club, Membership
from treasury.models import RemittanceType, Remittance, SpecialTransactionProxy
"""
Script d'import de la nk15:
TODO: import transactions
TODO: import adhesion
TODO: import activite
TODO: import ...
"""
M_DURATION = 396
M_START = datetime.date(2019, 8, 31)
M_END = datetime.date(2020, 9, 30)
MAP_IDBDE = {
-4: 2, # Carte Bancaire
-3: 4, # Virement
-2: 1, # Especes
-1: 3, # Chèque
0: 5, # BDE
}
MAP_IDACTIVITY = {}
MAP_NAMEACTIVITY = {}
MAP_NAMEGUEST = {}
MAP_IDSPECIALTRANSACTION = {}
def update_line(n, total, content):
n = str(n)
total = str(total)
n.rjust(len(total))
print(f"\r ({n}/{total}) {content:10.10}", end="")
@transaction.atomic
def import_comptes(cur):
cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
pkclub = 3
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["pseudo"])
if row["type"] == "personne":
# sanitize password
if row["passwd"] != "*|*" and not row["deleted"]:
passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
else:
passwd_nk15 = ''
try:
obj_dict = {
"username": row["pseudo"],
"password": passwd_nk15,
"first_name": row["nom"],
"last_name": row["prenom"],
"email": row["mail"],
"is_active": True, # temporary
}
user = User.objects.create(**obj_dict)
profile = user.profile
profile.phone_number = row['tel']
profile.address = row['adresse']
profile.paid = row['normalien']
profile.registration_valid = True
profile.email_confirmed = True
user.save()
profile.save()
# sanitize duplicate aliases (nk12)
except ValidationError as e:
if e.code == 'same_alias':
user.username = row["pseudo"] + str(row["idbde"])
user.save()
else:
raise e
# profile and note created via signal.
note = user.note
date = row.get("last_negatif", None)
if date is not None:
note.last_negative = make_aware(date)
note.balance = row["solde"]
note.save()
else: # club
obj_dict = {
"pk": pkclub,
"name": row["pseudo"],
"email": row["mail"],
"membership_duration": M_DURATION,
"membership_start": M_START,
"membership_end": M_END,
"membership_fee_paid": 0,
"membership_fee_unpaid": 0,
}
club, c = Club.objects.get_or_create(**obj_dict)
pkclub += 1
note = club.note
note.balance = row["solde"]
club.save()
note.save()
MAP_IDBDE[row["idbde"]] = note.note_ptr_id
@transaction.atomic
def import_boutons(cur):
cur.execute("SELECT * FROM boutons;")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["label"])
cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"])
if created:
cat.save()
obj_dict = {
"pk": row["id"],
"name": row["label"],
"amount": row["montant"],
"destination_id": MAP_IDBDE[row["destinataire"]],
"category": cat,
"display": row["affiche"],
"description": row["description"],
}
try:
with transaction.atomic(): # required for error management
button = TransactionTemplate.objects.create(**obj_dict)
except IntegrityError as e:
# button with the same name is not possible in NK20.
if "unique" in e.args[0]:
qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name')
note_name = qs[0]["name"]
# rename button name
obj_dict["name"] = f"{obj_dict_name['name']} {note_name}"
button = TransactionTemplate.objects.create(**obj_dict)
else:
raise e
button.save()
@transaction.atomic
def import_transaction(cur):
idmin = 58770
bde = Club.objects.get(name="BDE")
kfet = Club.objects.get(name="Kfet")
cur.execute(
"SELECT t.date AS transac_date, t.type, t.emetteur,\
t.destinataire,t.quantite, t.montant, t.description,\
t.valide, t.cantinvalidate, t.categorie, \
a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
FROM transactions AS t \
LEFT JOIN adhesions AS a ON t.id = a.idtransaction \
WHERE t.id> {} \
ORDER BY t.id;".format(idmin)
)
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["description"])
try:
date = make_aware(row["transac_date"])
except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))
# standart transaction object
obj_dict = {
# "pk": row["id"],
"destination_id": MAP_IDBDE[row["destinataire"]],
"source_id": MAP_IDBDE[row["emetteur"]],
"created_at": date,
"amount": row["montant"],
"quantity": row["quantite"],
"reason": row["description"],
"valid": row["valide"],
}
ttype = row["type"]
if ttype == "don" or ttype == "transfert":
Transaction.objects.create(**obj_dict)
elif ttype == "bouton":
cat_name = row["categorie"]
if cat_name is None:
cat_name = 'None'
cat, created = TemplateCategory.objects.get_or_create(name=cat_name)
if created:
cat.save()
obj_dict["category"] = cat
RecurrentTransaction.objects.create(**obj_dict)
elif ttype == "crédit" or ttype == "retrait":
field_id = "source_id" if ttype == "crédit" else "destination_id"
if "espèce" in row["description"]:
obj_dict[field_id] = 1
elif "carte" in row["description"]:
obj_dict[field_id] = 2
elif "cheques" in row["description"]:
obj_dict[field_id] = 3
elif "virement" in row["description"]:
obj_dict[field_id] = 4
pk = max(row["destinataire"], row["emetteur"])
actor = Note.objects.get(id=MAP_IDBDE[pk])
# custom fields of SpecialTransaction
if actor.__class__.__name__ == "NoteUser":
obj_dict["first_name"] = actor.user.first_name
obj_dict["last_name"] = actor.user.last_name
elif actor.__class__.__name__ == "NoteClub":
obj_dict["first_name"] = actor.club.name
obj_dict["last_name"] = actor.club.name
else:
raise Exception("Badly formatted Special Transaction You should'nt be there.")
tr = SpecialTransaction.objects.create(**obj_dict)
if "cheques" in row["description"]:
MAP_IDSPECIALTRANSACTION[row["id"]] = tr
elif ttype == "adhésion":
montant = row["montant"]
# Create Double membership to Kfet and Bde
# sometimes montant = 0, fees are modified accordingly.
bde_dict = {
"user": MAP_IDBDE[row["idbde"]],
"club": bde,
"date_start": row["date"].date(), # Only date, not time
"fee": min(500, montant)
}
kfet_dict = {
"user": MAP_IDBDE[row["idbde"]],
"club": kfet,
"date_start": row["date"].date(), # Only date, not time
"fee": max(montant - 500, 0),
}
if row["valide"]:
with transaction.atomic():
# membership save triggers MembershipTransaction creation
bde_membership = Membership.objects.get_or_create(**bde_dict)
kfet_membership = Membership.objects.get_or_create(**kfet_dict)
bde_membership.transaction.created_at = row["transac_date"]
bde_membership.transaction.description = row["description"]
bde_membership.transaction.save()
kfet_membership.transaction.created_at = row["transac_date"]
kfet_membership.transaction.description = row["description"] + "(Kfet)"
kfet_membership.transaction.save()
else:
# don't create membership
MembershipTransaction.objects.create(**obj_dict)
elif ttype == "invitation":
m = re.search(r"Invitation (.*?) \((.*?)\)", row["description"])
if m is None:
raise IntegrityError(f"Invitation is not well formated: {row['description']} (must be 'Invitation ACTIVITY_NAME (NAME)')")
activity_name = m.group(1)
guest_name = m.group(2)
if activity_name not in MAP_NAMEACTIVITY:
raise IntegrityError(f"Activity {activity_name} is not found")
activity = MAP_NAMEACTIVITY[activity_name]
if guest_name not in MAP_NAMEGUEST:
raise IntegrityError(f"Guest {guest_name} is not found")
guest = None
for g in MAP_NAMEGUEST[guest_name]:
if g.activity.pk == activity.pk:
guest = g
break
if guest is None:
raise IntegrityError("Guest {guest_name} didn't go to the activity {activity_name}")
obj_dict["guest"] = guest
GuestTransaction.objects.get_or_create(**obj_dict)
else:
print("other type not supported yet:", ttype)
@transaction.atomic
def import_aliases(cur):
cur.execute("SELECT * FROM aliases ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["alias"])
alias_name = row["alias"]
alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
obj_dict = {
"note_id": MAP_IDBDE[row["idbde"]],
"name": alias_name_good,
"normalized_name": Alias.normalize(alias_name_good),
}
try:
with transaction.atomic():
alias, created = Alias.objects.get_or_create(**obj_dict)
except IntegrityError as e:
if "unique" in e.args[0]:
continue
else:
raise e
alias.save()
@transaction.atomic
def import_activities(cur):
cur.execute("SELECT * FROM activites ORDER by id")
n = cur.rowcount
activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually
kfet = Club.objects.get(name="Kfet")
for idx, row in enumerate(cur):
update_line(idx, n, row["alias"])
organizer = Club.objects.filter(name=row["signature"])
if organizer.exists():
# Try to find the club that organizes the activity. If not found, assume it's Kfet (fix manually)
organizer = organizer.get()
else:
organizer = kfet
obj_dict = {
"name": row["titre"],
"description": row["description"],
"activity_type": activity_type, # By default Pot
"creater": MAP_IDBDE[row["responsable"]],
"organizer": organizer,
"attendees_club": kfet, # Maybe fix manually
"date_start": row["debut"],
"date_end": row["fin"],
"valid": row["validepar"] is not None,
"open": row["open"], # Should always be False
}
# WARNING: Fields lieu, liste, listeimprimee are missing
try:
with transaction.atomic():
activity = Activity.objects.get_or_create(**obj_dict)[0]
MAP_IDACTIVITY[row["id"]] = activity
MAP_NAMEACTIVITY[activity.name] = activity
except IntegrityError as e:
raise e
@transaction.atomic
def import_activity_entries(cur):
map_idguests = {}
cur.execute("SELECT * FROM invites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
obj_dict = {
"activity": MAP_IDACTIVITY[row["activity"]],
"last_name": row["nom"],
"first_name": row["prenom"],
"inviter": MAP_IDBDE[row["responsable"]],
}
try:
with transaction.atomic():
guest = Guest.objects.get_or_create(**obj_dict)[0]
map_idguests.setdefault(row["responsable"], [])
map_idguests[row["id"]].append(guest)
guest_name = guest.first_name + " " + guest.last_name
MAP_NAMEGUEST.setdefault(guest_name, [])
MAP_NAMEGUEST[guest_name].append(guest)
except IntegrityError as e:
raise e
cur.execute("SELECT * FROM entree_activites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
activity = MAP_IDACTIVITY[row["activity"]]
guest = None
if row["est_invite"]:
for g in map_idguests[row["id"]]:
if g.activity.pk == activity.pk:
guest = g
break
if not guest:
raise IntegrityError("Guest was not found: " + str(row))
obj_dict = {
"activity": activity,
"time": row["heure_entree"],
"note": guest.inviter if guest else MAP_IDBDE[row["idbde"]],
"guest": guest,
}
try:
with transaction.atomic():
Entry.objects.get_or_create(**obj_dict)
except IntegrityError as e:
raise e
@transaction.atomic
def import_remittances(cur):
cur.execute("SELECT * FROM remises ORDER by id")
map_idremittance = {}
n = cur.rowcount
check_type = RemittanceType.objects.get(note__name="Chèque")
for idx, row in enumerate(cur):
update_line(idx, n, row["date"])
obj_dict = {
"date": row["date"][10:],
"remittance_type": check_type,
"comment": row["commentaire"],
"closed": row["close"],
}
try:
with transaction.atomic():
remittance = Remittance.objects.get_or_create(**obj_dict)
map_idremittance[row["id"]] = remittance
except IntegrityError as e:
raise e
print("remittances are imported")
print("imported checks")
cur.execute("SELECT * FROM cheques ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["date"])
obj_dict = {
"date": row["date"][10:],
"remittance_type": check_type,
"comment": row["commentaire"],
"closed": row["close"],
}
tr = MAP_IDSPECIALTRANSACTION[row["idtransaction"]]
proxy = SpecialTransactionProxy.objects.get_or_create(transaction=tr)
proxy.remittance = map_idremittance[row["idremise"]]
try:
with transaction.atomic():
proxy.save()
except IntegrityError as e:
raise e
class Command(BaseCommand): class Command(BaseCommand):
""" """
@@ -460,60 +10,9 @@ class Command(BaseCommand):
Need to be run by a user with a registered role in postgres for the database nk15. Need to be run by a user with a registered role in postgres for the database nk15.
""" """
def print_success(self, to_print):
return self.stdout.write(self.style.SUCCESS(to_print))
def add_arguments(self, parser):
parser.add_argument('-c', '--comptes', action='store_true', help="import accounts")
parser.add_argument('-b', '--boutons', action='store_true', help="import boutons")
parser.add_argument('-t', '--transactions', action='store_true', help="import transaction")
parser.add_argument('-al', '--aliases', action='store_true', help="import aliases")
parser.add_argument('-ac', '--activities', action='store_true', help="import activities")
parser.add_argument('-r', '--remittances', action='store_true', help="import check remittances")
parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name')
parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner')
def handle(self, *args, **kwargs): def handle(self, *args, **kwargs):
global MAP_IDBDE subprocess.call("./apps/scripts/shell/tabularasa")
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] call_command('import_account', alias=True, chunk=1000, save = "map.json")
# connecting to nk15 database call_command('import_activities', chunk=100, map="map.json")
conn = pg.connect(database=nk15db, user=nk15user) call_command('import_transaction', buttons=True, map="map.json")
cur = conn.cursor(cursor_factory=pge.DictCursor) #
if kwargs["comptes"]:
# reset database.
call_command("migrate")
call_command("loaddata", "initial")
self.print_success("reset nk20 database\n")
import_comptes(cur)
self.print_success("comptes table imported")
elif kwargs["map"]:
filename = kwargs["map"]
with open(filename, 'r') as fp:
MAP_IDBDE = json.load(fp)
MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()}
if kwargs["save"]:
filename = kwargs["save"]
with open(filename, 'w') as fp:
json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
# /!\ need a prober MAP_IDBDE
if kwargs["boutons"]:
import_boutons(cur)
self.print_success("boutons table imported\n")
if kwargs["activities"]:
import_activities(cur)
self.print_success("activities imported\n")
import_activity_entries(cur)
self.print_success("activity entries imported\n")
if kwargs["aliases"]:
import_aliases(cur)
self.print_success("aliases imported\n")
if kwargs["transactions"]:
import_transaction(cur)
self.print_success("transaction imported\n")
if kwargs["remittances"]:
import_remittances(cur)
self.print_success("remittances imported\n")