Created
January 4, 2023 15:31
-
-
Save devoNOTbevo/0909bde70515dd56d6e4d98895f83202 to your computer and use it in GitHub Desktop.
A beginner friendly
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const mysql = require('mysql'); | |
const fs = require('graceful-fs'); | |
// Make any queries constant for readability | |
const TABLES_QUERY = | |
"SELECT table_name FROM information_schema.tables WHERE table_schema ='my_table_schema"; | |
const COLUMNS_QUERY = 'SELECT column_name from information_schema.columns'; | |
// Create the MySQL connection | |
const connection = mysql.createConnection({ | |
host: 'localhost', | |
user: 'root', | |
password: '', | |
database: 'my_table_schema', | |
}); | |
// Create an IIFE so we can do async/await | |
(async () => { | |
try { | |
// get table names first | |
const tables: string[] = await connectToDBAndGetTables(); | |
// now loop through them | |
for (let table of tables) { | |
const tableData: string[] = await getTableData(table); | |
console.log('writing file for ' + table); | |
let curFileName = `${__dirname}/data/${table}.csv`; | |
// write/append line by line | |
for (let datum of tableData) { | |
await writeDataLine(curFileName, datum + '\r\n'); | |
} | |
// done with this table | |
console.log('done with ' + table + '\n\n'); | |
} | |
// make sure to close connection if we're done | |
connection.end(); | |
} catch (e) { | |
console.log(e); | |
connection.end(); | |
} | |
})(); | |
// promisifies connection to MySQL | |
// and getting tables | |
async function connectToDBAndGetTables(): Promise<string[]> { | |
return new Promise((resolve, reject) => { | |
// CONNECT TO THE DATABASE | |
connection.connect((connectionError) => { | |
if (connectionError) reject(connectionError); | |
console.log('Connected to MySQL Server!'); | |
// GET ALL THE TABLES | |
connection.query(TABLES_QUERY, (tablesError, tablesResults) => { | |
if (tablesError) reject(tablesError); | |
console.log('Got the Tables....'); | |
console.log('now getting the columns of each....'); | |
resolve(tablesResults.map((tablesResult) => tablesResult.TABLE_NAME)); | |
}); | |
}); | |
}); | |
} | |
// promisifies getting the data from the table | |
// this also retains the column names as the first | |
// element, allowing you to save a header row to the CSV | |
async function getTableData(curTableName): Promise<string[]> { | |
console.log('getting results for: ' + curTableName); | |
return new Promise((resolve, reject) => { | |
// GET THE COLUMNS FOR THE TABLE | |
connection.query( | |
`${COLUMNS_QUERY} WHERE table_schema='ccosa_test' and table_name = '${curTableName}';`, | |
(columnsError, columnsResults) => { | |
if (columnsError) reject(columnsError); | |
let headers = columnsResults.map((column) => column.COLUMN_NAME); | |
// GET THE DATA NOW | |
connection.query( | |
`SELECT ${headers.join(',')} FROM ${curTableName}`, | |
(rowsError, rowsResults) => { | |
if (rowsError) reject(rowsError); | |
const formattedRowsResults = rowsResults.map((rowsResult) => | |
Object.values(rowsResult) | |
); | |
const allData = [headers, ...formattedRowsResults].map((arr) => | |
arr.join(',') | |
); | |
resolve(allData); | |
} | |
); | |
} | |
); | |
}); | |
} | |
// uses fs appending to write a line of data | |
// in a promisified way. | |
async function writeDataLine(file, data) { | |
return new Promise((resolve, reject) => { | |
fs.appendFile(file, data, (lineError) => { | |
if (lineError) reject(lineError); | |
resolve(true); | |
}); | |
}); | |
} |
Is Promise<string[]>
a Promise with a generic string array?
https://www.typescriptlang.org/docs/handbook/2/generics.html
typescriptlang.orgtypescriptlang.org
Documentation - Generics
Types which take parameters
generic string array (self-coined term) as a term seems super weird . In C I am fine with char arrays holding strings, but in C++ string is formed with a constructor and in memory it may actually be an array but [ ]
is not used IIRC. Javascript isn't Object oriented like C++ or Java but it can be made to appear like it al la typescript IIRC
.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
A few things to note:
connection.query
, but this led to a memory leak and it would write the lines out of order. to keep the order of the database and also ensure the headers are at the top, I had to promisify everything manually.writeDataLine
) is because my first approach wrote all of the table contents to a file in one go. This caused the resulting string CSV-style string to exceed the size that JS allows. See more here. Sincefs
can append line by line reasonably anyways, I went that route.