sample code

This commit is contained in:
2022-10-13 11:55:19 -04:00
commit bde8aa5e5d
67 changed files with 6421 additions and 0 deletions
@@ -0,0 +1,10 @@
name: Dafter Import CSV
description: Allows user to import transactions from a CSV file.
core: 8.x
type: module
package: Daftar
core_version_requirement: ^8 || ^9
version: 1.0.0
dependencies:
- daftar_content:daftar_content
- daftar_stock_api:daftar_stock_api
@@ -0,0 +1,15 @@
daftar_import.csv_orders:
path: '/import/csv/orders'
defaults:
_form: '\Drupal\daftar_import\Form\ImportCsvOrdersForm'
_title: 'Import Orders from CSV'
requirements:
_role: 'authenticated'
daftar_import.csv_transfers:
path: '/import/csv/transfers'
defaults:
_form: '\Drupal\daftar_import\Form\ImportCsvTransfersForm'
_title: 'Import Transfers from CSV'
requirements:
_role: 'authenticated'
@@ -0,0 +1,10 @@
<?php
namespace Drupal\daftar_import\Exception;
use Exception;
class CsvDataException extends Exception {
}
@@ -0,0 +1,557 @@
<?php
namespace Drupal\daftar_import\Form;
use Drupal\Component\Utility\Bytes;
use Drupal\Component\Utility\Environment;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\Core\Form\FormBase;
use Drupal\Core\Form\FormState;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Messenger\MessengerInterface;
use Drupal\daftar_content\Form\AccountForm;
use Drupal\daftar_content\Form\ContentBaseForm;
use Drupal\daftar_import\Exception\CsvDataException;
use Drupal\daftar_stock_api\Exception\StockApiException;
use Drupal\daftar_stock_api\StockApiServiceInterface;
use Drupal\file\FileInterface;
use Drupal\file\FileStorageInterface;
use Drupal\node\NodeInterface;
use Drupal\node\NodeStorageInterface;
use Drupal\taxonomy\TermStorageInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Provides shared methods.
* @package Drupal\daftar\Form
*/
abstract class ImportCsvBaseForm extends FormBase {
const MAX_CSV_FILE_SIZE = '5MB';
const TERM_TYPE_SYMBOLS = 'symbols';
const TERM_TYPE_EXCHANGES = 'exchanges';
const TERM_TYPE_SECTORS = 'sectors';
const TERM_TYPE_INDUSTRIES = 'industries';
/**
* @var NodeStorageInterface
*/
protected $nodeStorage;
/**
* @var TermStorageInterface
*/
protected $termStorage;
/**
* @var FileStorageInterface
*/
protected $fileStorage;
/**
* @var StockApiServiceInterface
*/
protected $stockApi;
/**
* @var AccountForm
*/
protected $accountForm;
public function __construct(NodeStorageInterface $node_storage, TermStorageInterface $term_storage, FileStorageInterface $file_storage, StockApiServiceInterface $stock_api) {
$this->nodeStorage = $node_storage;
$this->termStorage = $term_storage;
$this->fileStorage = $file_storage;
$this->stockApi = $stock_api;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container) {
$form = new static(
$container->get('entity_type.manager')->getStorage('node'),
$container->get('entity_type.manager')->getStorage('taxonomy_term'),
$container->get('entity_type.manager')->getStorage('file'),
$container->get('daftar.stock_api.alphavantage')
);
$form->accountForm = AccountForm::create($container);
return $form;
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state) {
$brokerages = $this->getBrokerageOptions();
$maxCsvSize = intval($this->getMaxCsvSize());
$form['description'] = [
'#type' => 'item',
'#markup' => '<strong>Override or unset this element.</strong>',
'#weight' => '-20',
];
if (!empty($brokerages)) {
$form['brokerage'] = [
'#type' => 'select',
'#title' => $this->t('Brokerage Account'),
'#options' => $brokerages,
'#required' => TRUE,
'#validated' => TRUE, // Setting it to TRUE stops default validation. Must validate manually.
'#weight' => -15,
];
}
else {
$form['brokerage'] = [
'#type' => 'hidden',
'#value' => 'new'
];
}
$form += $this->buildAccountForm($form_state);
if (isset($form['title'])) {
unset($form['title']['#required']);
$form['title']['#title'] = $this->t('New Account Alias');
$form['title']['#weight'] = -10;
$form['title']['#states'] = [
'visible' => [
':input[name="brokerage"]' => ['value' => 'new'],
],
'required' => [
':input[name="brokerage"]' => ['value' => 'new'],
]
];
}
$form['csv_file'] = [
'#type' => 'managed_file',
'#title' => $this->t('CSV File'),
'#description' => $this->t(
'Upload a CSV file containing your transactions. Maximum file size allowed: %size',
['%size' => $this->getReadableFileSize($maxCsvSize)]
),
'#multiple' => FALSE,
'#upload_location' => 'private://imports',
'#upload_validators' => [
'file_validate_extensions' => ['csv'],
'file_validate_size' => [$maxCsvSize],
],
'#required' => TRUE,
];
$form['file_description'] = [
'#type' => 'item',
'#markup' => $this->t(
'Required columns in the CSV: %columns.',
['%columns' => $this->getNaturalLanguageJoin($this->getRequiredKeys())]
),
];
// Add a submit button that handles the submission of the form.
$form['actions']['submit'] = [
'#type' => 'submit',
'#value' => $this->t('Import'),
];
return $form;
}
protected function buildAccountForm(FormStateInterface $form_state) {
$accountFormState = new FormState();
$accountForm = $this->accountForm->buildForm([], $accountFormState, 'brokerage', ContentBaseForm::DISPLAY_MODE_EMBEDDED);
$form_state->set('account_form', $accountForm);
$form_state->set('account_form_state', $accountFormState);
return $accountForm;
}
/**
* {@inheritdoc}
*/
public function validateForm(array &$form, FormStateInterface $form_state) {
$brokerages = $this->getBrokerageOptions();
$brokerageSelected = $form_state->getValue('brokerage');
if ($brokerageSelected === 'new') {
$this->validateAccountForm($form_state);
}
// Make sure the selected brokerage account belongs to the current user.
elseif ($brokerageSelected !== 'new' && !isset($brokerages[$brokerageSelected])) {
$form_state->setErrorByName('brokerage', $this->t('Please select a brokerage account.'));
}
}
protected function validateAccountForm(FormStateInterface $form_state) {
/** @var FormStateInterface $accountFormState */
$newBrokerageTitle = trim($form_state->getValue('title'));
$accountForm = $form_state->get('account_form');
$accountFormState = $form_state->get('account_form_state');
$accountFormState->setValue('title', $newBrokerageTitle);
$accountFormState->setValue('type', 'brokerage');
$this->accountForm->validateForm($accountForm, $accountFormState);
$accountFormErrors = (array) $accountFormState->getErrors();
foreach ($accountFormErrors as $key => $error) {
$form_state->setErrorByName($key, $error);
}
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state) {
/** @var NodeInterface $brokerage */
$brokerageSelected = $form_state->getValue('brokerage');
if ($brokerageSelected === 'new') {
$brokerage = $this->submitAccountForm($form_state);
}
else {
$brokerage = $this->getBrokerageNodeFromUuid($brokerageSelected);
}
if (NULL === $brokerage) {
$this->messenger()->addMessage(
$this->t('Unable to retrieve brokerage account.'),
MessengerInterface::TYPE_ERROR
);
return FALSE;
}
$csvFile = $form_state->getValue('csv_file');
if (isset($csvFile[0])) {
try {
$file = $this->preserveFile($csvFile[0]);
$data = $this->parseCsvToArray($file->getFileUri());
$this->messenger()->addMessage(
$this->t('Hooray! Passed Validation!'),
MessengerInterface::TYPE_STATUS
);
return [
'brokerage' => $brokerage,
'data' => $data,
];
}
catch (EntityStorageException $e) {
$this->messenger()->addMessage(
$this->t('Unable to process CSV file. Please try again.'),
MessengerInterface::TYPE_ERROR
);
}
catch (CsvDataException $e) {
$this->messenger()->addMessage(
['#type' => 'item', '#markup' => $e->getMessage()],
MessengerInterface::TYPE_ERROR
);
}
}
return FALSE;
}
protected function submitAccountForm(FormStateInterface $form_state) {
$accountForm = $form_state->get('account_form');
$accountFormState = $form_state->get('account_form_state');
$this->accountForm->submitForm($accountForm, $accountFormState);
return $accountFormState->get('account_node');
}
/**
* Remove whitespaces form a string and converts it to lowercase.
* @param string $key a string to sanitize
* @return string a sanitized string
*/
public static function sanitizeKey($key) {
$key = trim($key);
return strtolower($key);
}
/**
* @return array an array of required keys
*/
protected abstract function getRequiredKeys();
protected function getMaxCsvSize() {
static $max_size = -1;
if ($max_size < 0) {
// Start with site-wide max size.
$max_size = Environment::getUploadMaxSize();
// Reduce max_size if MAX_CSV_FILE_SIZE is less. Except if
// MAX_CSV_FILE_SIZE is zero, which indicates no limit.
$upload_max = Bytes::toInt(self::MAX_CSV_FILE_SIZE);
if ($upload_max > 0 && $upload_max < $max_size) {
$max_size = $upload_max;
}
}
return $max_size;
}
protected function getReadableFileSize($size, $precision = 2) {
$units = array('B','kB','MB','GB','TB','PB','EB','ZB','YB');
$step = 1024;
$i = 0;
while (($size / $step) > 0.9) {
$size = $size / $step;
$i++;
}
return round($size, $precision).$units[$i];
}
protected function getNaturalLanguageJoin(array $list, $conjunction = 'and') {
$last = array_pop($list);
if ($list) {
return implode(', ', $list) . ' ' . $conjunction . ' ' . $last;
}
return $last;
}
protected function getBrokerageOptions() {
$brokerages = [];
$results = $this->getBrokerageNids();
if (!empty($results)) {
$brokerages['new'] = '- ' . $this->t('Create a brokerage account') . ' -';
$nodes = $this->nodeStorage->loadMultiple($results);
foreach ($nodes as $nid => $node) {
$label = (boolean) $node->field_account_status->value
? $node->label()
: $node->label() . ' (inactive)';
$brokerages[$node->uuid()] = $label;
}
}
return $brokerages;
}
protected function getBrokerageNids() {
return $this->nodeStorage->getQuery()
->condition('status', 1)
->condition('type', 'account')
->condition('uid', $this->currentUser()->id())
->condition('field_account_type', 'br')
->execute();
}
/**
* @param string $uuid
* @return NodeInterface|null
*/
protected function getBrokerageNodeFromUuid($uuid) {
/** @var NodeInterface $brokerage */
$brokerage = NULL;
$results = $this->nodeStorage->getQuery()
->condition('status', 1)
->condition('type', 'account')
->condition('uuid', $uuid)
->condition('field_account_type', 'br')
->range(0, 1)
->execute();
if (!empty($results)) {
reset($results);
$brokerage = $this->nodeStorage->load(key($results));
}
return $brokerage;
}
/**
* @param $fid
* @return FileInterface|null
* @throws EntityStorageException
*/
protected function preserveFile($fid) {
/** @var FileInterface $file */
$file = $this->fileStorage->load($fid);
$file->setPermanent();
$file->save();
return $file;
}
/**
* @param $file_uri
* @return array an associative array of CSV rows indexed by header keys
* @throws EntityStorageException
* @throws CsvDataException
*/
protected function parseCsvToArray($file_uri) {
if(!file_exists($file_uri) || !is_readable($file_uri)) {
throw new EntityStorageException('File does not exists or is not readable.');
}
$header = NULL;
$rows = [];
if (($handle = fopen($file_uri, 'r')) !== FALSE ) {
while (($row = fgetcsv($handle, 1000)) !== FALSE) {
if (!$header){
// Sanitize keys in the header.
$header = array_map([__CLASS__, 'sanitizeKey'], $row);
// Check for missing keys.
$missingRequiredKeys = array_diff($this->getRequiredKeys(), $header);
if (!empty($missingRequiredKeys)) {
throw new CsvDataException($this->t(
'Missing required columns: %columns',
['%columns' => implode($missingRequiredKeys, ', ')]
));
}
continue;
}
$associativeRow = [];
foreach ($row as $i => $value) {
$associativeRow[$header[$i]] = $value;
}
$rows[] = $associativeRow;
}
fclose($handle);
}
return $rows;
}
/**
* Parses a date string and converts it to ISO8601 format.
* @param string $date_string A date string.
* @return string A date string in ISO8601 format.
* @throws CsvDataException
* @todo Maybe this function can be swapped out with a Drupal/PHP function.
*/
protected function getIso8601DateString($date_string) {
$parsedDate = date_parse($date_string);
if (FALSE === $parsedDate) {
throw new CsvDataException('Unable to parse date.');
}
if (!isset($parsedDate['year'])) {
throw new CsvDataException('Date is missing the year.');
}
$date = $parsedDate['year'];
if (isset($parsedDate['month'])) {
$date .= '-' . str_pad($parsedDate['month'], 2, '0', STR_PAD_LEFT);
}
if (isset($parsedDate['day'])) {
$date .= '-' . str_pad($parsedDate['day'], 2, '0', STR_PAD_LEFT);
}
return $date;
}
/**
* Find the taxonomy term ID of a given term.
*
* @param string $name The term name.
* @param string $vid The vocabulary ID.
* @param bool $create_term A boolean flag indicating whether to create
* a new term when not found.
*
* @return int|null Returns the term ID for a given term; null otherwise.
*
* @throws EntityStorageException
* @throws CsvDataException
* @throws StockApiException
*/
protected function getTermId($name, $vid, $create_term = FALSE) {
// Extract symbol from
if ($vid === static::TERM_TYPE_SYMBOLS) {
$symbol = strtoupper($name);
if (strpos($symbol, ':') !== FALSE) {
$symbolSplit = explode(':', $symbol);
if (!$symbolSplit[1]) {
throw new CsvDataException('Invalid symbol provided.');
}
$name = $symbolSplit[1];
}
}
$termIds = $this->termStorage->getQuery()
->condition('vid', $vid)
->condition('name', $name)
->range(0, 1)
->execute();
if (!empty($termIds)) {
return reset($termIds);
}
if ($create_term) {
return $this->createTerm($name, $vid);
}
return NULL;
}
/**
* Creates the taxonomy term entity for a given term.
*
* @param string $name The term name.
* @param string $vid The vocabulary ID.
*
* @return int|null Returns the term ID for a given term; null otherwise.
*
* @throws EntityStorageException
* @throws CsvDataException
* @throws StockApiException
*/
protected function createTerm($name, $vid) {
$data = [
'name' => $name,
'vid' => $vid,
];
if ($vid === static::TERM_TYPE_SYMBOLS) {
$profile = $this->stockApi->doSymbolLookup($name);
if (empty($profile)) {
return NULL;
}
$exchangeTid = $this->getTermId($profile['Exchange'], static::TERM_TYPE_EXCHANGES, TRUE);
$sectorTid = $this->getTermId($profile['Sector'], static::TERM_TYPE_SECTORS, TRUE);
$industryTid = $this->getTermId($profile['Industry'], static::TERM_TYPE_INDUSTRIES, TRUE);
$data += [
'field_company_name' => $profile['Name'],
'field_exchange' => ['target_id' => $exchangeTid],
'field_sectors' => ['target_id' => $sectorTid],
'field_industries' => ['target_id' => $industryTid],
];
}
$term = $this->termStorage->create($data);
$term->save();
return $term->id();
}
}
@@ -0,0 +1,130 @@
<?php
namespace Drupal\daftar_import\Form;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Form\FormStateInterface;
use Drupal\daftar_content\Entity\Order;
use Drupal\daftar_import\Exception\CsvDataException;
use Drupal\daftar_stock_api\Exception\StockApiException;
/**
* Implements form for processing order transactions from a CSV file.
* @package Drupal\daftar\Form
*/
class ImportCsvOrdersForm extends ImportCsvBaseForm {
/**
* @var EntityStorageInterface
*/
protected $orderStorage;
/**
* {@inheritdoc}
*/
public function getFormId() {
return 'daftar_csv_import_form';
}
/**
* @inheritDoc
*/
protected function getRequiredKeys() {
return [
'id',
'date',
'action',
'symbol',
'quantity',
'amount',
];
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state) {
$form = parent::buildForm($form, $form_state);
$form['#title'] = 'Import Orders from CSV';
$form['description']['#markup'] = $this->t('Import transactions such as Buy, Sell, Sell Short and Buy to Cover from your brokerage.');
return $form;
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state) {
$data = parent::submitForm($form, $form_state);
// End processing if no CSV data was retrieved.
if (FALSE === $data) {
return;
}
$currentRow = 1; // Starts at 1 because first row is a header row.
try {
$this->orderStorage = \Drupal::entityTypeManager()->getStorage('transaction_order');
$brokerageNodeId = $data['brokerage']->id();
foreach ($data['data'] as $row) {
$currentRow++;
$this->createOrder($brokerageNodeId, $row);
}
}
catch(CsvDataException $e) {
$this->messenger->addError(
'Data error on row ' . $currentRow . ': ' . $e->getMessage()
);
}
catch (\Exception $e) {
// @TODO log to watchdog
$this->messenger()->addError(
"Import process stopped at row {$currentRow} due to internal errors."
);
}
}
/**
* Creates the order entity.
*
* @param int $aidBrokerage Node ID of the brokerage account.
* @param array $data An associative array containing CSV row.
*
* @throws CsvDataException
* @throws EntityStorageException
* @throws StockApiException
*/
protected function createOrder($aidBrokerage, array $data) {
$dateIso8601 = $this->getIso8601DateString($data['date']);
$symbolTid = $this->getTermId($data['symbol'], static::TERM_TYPE_SYMBOLS, TRUE);
$action = Order::mapAction($data['action']);
if (!$action) {
throw new CsvDataException('Unable to parse order action.');
}
$type = Order::mapType($data['type']);
if (!$type) {
throw new CsvDataException('Unable to parse order type.');
}
$this->orderStorage->create([
'aid' => $aidBrokerage,
'transaction_date' => $dateIso8601,
'symbol' => $symbolTid,
'action' => $action,
'type' => $type,
'amount' => $data['amount'],
'quantity' => $data['quantity'],
'on_hand' => $data['quantity'],
'commission' => (isset($data['commission']) && $data['commission'] !== NULL) ? $data['commission'] : 0,
'fees' => (isset($data['fees']) && $data['fees'] !== NULL) ? $data['fees'] : 0,
'notes' => (isset($data['notes'])) ? $data['notes'] : NULL,
])->save();
}
}
@@ -0,0 +1,188 @@
<?php
namespace Drupal\daftar_import\Form;
use Drupal\Core\Entity\EntityStorageException;
use Drupal\Core\Form\FormState;
use Drupal\Core\Form\FormStateInterface;
use Drupal\Core\Messenger\MessengerInterface;
use Drupal\daftar_import\Exception\CsvDataException;
/**
* Implements form for processing transfer transactions from a CSV file.
* @package Drupal\daftar\Form
*/
class ImportCsvTransfersForm extends ImportCsvBaseForm {
/**
* @var \Drupal\Core\Entity\EntityStorageInterface
*/
protected $transferStorage;
/**
* {@inheritdoc}
*/
public function getFormId() {
return 'daftar_import_csv_transfer_form';
}
/**
* @inheritDoc
*/
protected function getRequiredKeys() {
return [
'id',
'date',
'amount',
'action',
'account',
];
}
/**
* {@inheritdoc}
*/
public function buildForm(array $form, FormStateInterface $form_state) {
$form = parent::buildForm($form, $form_state);
$form['#title'] = 'Import Transfers from CSV';
$form['description']['#markup'] = $this->t('Import transactions such as deposits and withdrawals.');
return $form;
}
/**
* {@inheritdoc}
*/
public function submitForm(array &$form, FormStateInterface $form_state) {
$data = parent::submitForm($form, $form_state);
// End processing if no CSV data was retrieved.
if (FALSE === $data) {
return;
}
$currentRow = 1; // Starts at 1 because first row is a header row.
try {
$this->transferStorage = \Drupal::entityTypeManager()->getStorage('transaction_transfer');
$brokerageNodeId = $data['brokerage']->id();
foreach ($data['data'] as $row) {
$currentRow++;
$this->createTransfer($brokerageNodeId, $row);
}
}
catch (CsvDataException $e) {
$this->messenger->addError(
'Data error on row ' . $currentRow . ': ' . $e->getMessage()
);
}
catch (\Exception $e) {
// @TODO log to watchdog
$this->messenger()->addError(
"Import process stopped at row {$currentRow} due to internal errors."
);
}
}
/**
* Find the bank account node ID of a given account alias.
*
* @param string $alias Alias for the bank account.
* @param bool $create_account A boolean flag indicating whether to create
* a new bank account when not found.
*
* @return int|null Returns the bank account node ID for a given alias; null otherwise.
* @throws \Exception
*/
protected function getBankNodeId($alias, $create_account = FALSE) {
$nodeIds = $this->nodeStorage->getQuery()
->condition('status', 1)
->condition('type', 'account')
->condition('uid', $this->currentUser()->id())
->condition('title', $alias)
->condition('field_account_type', 'ba')
->range(0, 1)
->execute();
if (!empty($nodeIds)) {
return reset($nodeIds);
}
if ($create_account) {
return $this->createBankAccount($alias);
}
return NULL;
}
/**
* Creates a bank account for the user.
*
* @param string $alias Alias for the bank account.
*
* @return int|null Returns the bank account node ID when successfully created; null otherwise.
*
* @throws CsvDataException
*/
protected function createBankAccount($alias) {
$accountFormState = new FormState();
$accountForm = $this->accountForm->buildForm([], $accountFormState, 'bank');
$accountFormState->setValue('title', $alias);
$accountFormState->setValue('type', 'bank');
$this->accountForm->validateForm($accountForm, $accountFormState);
$accountFormErrors = (array) $accountFormState->getErrors();
if (!empty($accountFormErrors)) {
throw new CsvDataException("Unable to create bank account: " . reset($accountFormErrors));
}
$this->accountForm->submitForm($accountForm, $accountFormState);
$node = $accountFormState->get('account_node');
if ($node) {
return $node->id();
}
return NULL;
}
/**
* Creates the transfer entity.
*
* @param int $aidBrokerage Node ID of the brokerage account.
* @param array $data An associative array containing CSV row.
*
* @throws CsvDataException
* @throws EntityStorageException
* @throws \Exception
*/
protected function createTransfer($aidBrokerage, $data) {
$dateIso8601 = $this->getIso8601DateString($data['date']);
$bankNodeId = $this->getBankNodeId($data['account'], TRUE);
if (!$bankNodeId) {
throw new \Exception("Unable to create bank account for alias {$data['account']}.");
}
if (strtolower($data['action']) === 'deposit') {
$aid = $aidBrokerage;
$accountWithdrawn = $bankNodeId;
}
else {
$aid = $bankNodeId;
$accountWithdrawn = $aidBrokerage;
}
$this->transferStorage->create([
'aid' => $aid,
'account_withdrawn' => $accountWithdrawn,
'amount' => $data['amount'],
'transaction_date' => $dateIso8601,
])->save();
}
}