diff options
author | Matthew Lemon <y@yulqen.org> | 2024-04-12 20:34:02 +0100 |
---|---|---|
committer | Matthew Lemon <y@yulqen.org> | 2024-04-12 20:34:02 +0100 |
commit | 85d77655a515c120e5c0ba3c48c6be4eda1ac8f6 (patch) | |
tree | b867aca6afc90b6b24859061e67ac63d87f7f860 | |
parent | 89ee67970d350e7381554cf38e464eda80b439ec (diff) |
First test and first appearance of returns code
Test a function that gets all sheets from a datamap struct.
A new function to handle a datamap and a return together in one POST
request.
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | cmd/dbasik-api/datamaps.go | 167 | ||||
-rw-r--r-- | cmd/dbasik-api/datamaps_test.go | 50 | ||||
-rw-r--r-- | cmd/dbasik-api/handlers.go | 256 | ||||
-rw-r--r-- | cmd/dbasik-api/routes.go | 1 | ||||
-rw-r--r-- | datamap.csv | 3 | ||||
-rw-r--r-- | go.mod | 18 | ||||
-rw-r--r-- | go.sum | 32 |
8 files changed, 371 insertions, 157 deletions
@@ -1,2 +1,3 @@ bin/ .env +resources/test.xlsm diff --git a/cmd/dbasik-api/datamaps.go b/cmd/dbasik-api/datamaps.go index 94bccbf..1121581 100644 --- a/cmd/dbasik-api/datamaps.go +++ b/cmd/dbasik-api/datamaps.go @@ -19,12 +19,7 @@ package main import ( "database/sql" - "encoding/csv" - "encoding/json" "errors" - "fmt" - "net/http" - "strconv" "time" ) @@ -69,6 +64,18 @@ type datamapModel struct { DB *sql.DB } +func GetSheetsFromDM(dm datamap) []string { + sheets := map[string]struct{}{} + for _, dml := range dm.DMLs { + sheets[dml.Sheet] = struct{}{} + } + var out []string + for sheet := range sheets { + out = append(out, sheet) + } + return out +} + func NewModels(db *sql.DB) Models { return Models{ Datamaps: datamapModel{DB: db}, @@ -113,153 +120,3 @@ func (m *datamapLineModel) Insert(dm datamap, dmls []datamapLine) (int, error) { tx.Commit() return int(datamapID), nil } - -func (app *application) createDatamapHandler(w http.ResponseWriter, r *http.Request) { - // Parse the multipart form - err := r.ParseMultipartForm(10 << 20) // 10Mb max - if err != nil { - app.serverErrorResponse(w, r, err) - } - - // Get form values - dmName := r.FormValue("name") - app.logger.Info("obtain value from form", "name", dmName) - dmDesc := r.FormValue("description") - app.logger.Info("obtain value from form", "description", dmDesc) - - // Get the uploaded file and name - file, _, err := r.FormFile("file") - if err != nil { - http.Error(w, "Missing file", http.StatusBadRequest) - return - } - defer file.Close() - - // parse the csv - reader := csv.NewReader(file) - var dmls []datamapLine - var dm datamap - - for { - line, err := reader.Read() - if err != nil { - if err.Error() == "EOF" { - break // end of file - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if len(line) != 4 { - http.Error(w, "Invalid CSV Format", http.StatusBadRequest) - return - } - - dmls = append(dmls, datamapLine{ - Key: line[0], - Sheet: line[1], - DataType: line[2], - Cellref: line[3], - }) - } - dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} - - err = app.writeJSONPretty(w, http.StatusOK, envelope{"datamap": dm}, nil) - if err != nil { - app.logger.Debug("writing out csv", "err", err) - app.serverErrorResponse(w, r, err) - } - - fmt.Fprintf(w, "file successfully uploaded") -} - -func (app *application) saveDatamapHandler(w http.ResponseWriter, r *http.Request) { - // Parse the multipart form - err := r.ParseMultipartForm(10 << 20) // 10Mb max - if err != nil { - app.serverErrorResponse(w, r, err) - } - // Get form values - dmName := r.FormValue("name") - app.logger.Info("obtain value from form", "name", dmName) - dmDesc := r.FormValue("description") - app.logger.Info("obtain value from form", "description", dmDesc) - - // Get the uploaded file and name - file, _, err := r.FormFile("file") - if err != nil { - http.Error(w, "Missing file", http.StatusBadRequest) - return - } - defer file.Close() - - // parse the csv - reader := csv.NewReader(file) - var dmls []datamapLine - var dm datamap - - for { - line, err := reader.Read() - if err != nil { - if err.Error() == "EOF" { - break // end of file - } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if len(line) != 4 { - http.Error(w, "Invalid CSV Format", http.StatusBadRequest) - return - } - - dmls = append(dmls, datamapLine{ - ID: 0, - Key: line[0], - Sheet: line[1], - DataType: line[2], - Cellref: line[3], - }) - - } - dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} - - // save to the database - _, err = app.models.DatamapLines.Insert(dm, dmls) - if err != nil { - http.Error(w, "Cannot save to database", http.StatusBadRequest) - return - } - -} - -func (app *application) getJSONForDatamap(w http.ResponseWriter, r *http.Request) { - // Get the DM out of the database - // dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} - - // err = app.writeJSONPretty(w, http.StatusOK, envelope{"datamap": dm}, nil) - // if err != nil { - // app.logger.Debug("writing out csv", "err", err) - // app.serverErrorResponse(w, r, err) - // } - - // fmt.Fprintf(w, "file successfully uploaded") -} - -func (app *application) showDatamapHandler(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - app.logger.Info("the id requested", "id", id) - id_int, err := strconv.ParseInt(id, 10, 64) - if err != nil || id_int < 1 { - app.notFoundResponse(w, r) - } - fmt.Fprintf(w, "show the details for datamap %d\n", id_int) -} - -func (app *application) createDatamapLine(w http.ResponseWriter, r *http.Request) { - var input datamapLine - err := json.NewDecoder(r.Body).Decode(&input) - if err != nil { - app.errorResponse(w, r, http.StatusBadRequest, err.Error()) - return - } - fmt.Fprintf(w, "%v\n", input) -} diff --git a/cmd/dbasik-api/datamaps_test.go b/cmd/dbasik-api/datamaps_test.go new file mode 100644 index 0000000..428f301 --- /dev/null +++ b/cmd/dbasik-api/datamaps_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "slices" + "testing" + "time" +) + +func TestReadDML(t *testing.T) { + dm := datamap{ + ID: 0, + Name: "Test Name", + Description: "Test description", + Created: time.Now(), + DMLs: []datamapLine{ + { + ID: 1, + Key: "Test Key", + Sheet: "Test Sheet", + DataType: "TEXT", + Cellref: "A10", + }, + { + ID: 2, + Key: "Test Key 2", + Sheet: "Test Sheet", + DataType: "TEXT", + Cellref: "A11", + }, + { + ID: 3, + Key: "Test Key 3", + Sheet: "Test Sheet 2", + DataType: "TEXT", + Cellref: "A12", + }, + }, + } + + got := GetSheetsFromDM(dm) + if !slices.Contains(got, "Test Sheet") { + t.Errorf("expected to find Test Sheet in %v but didn't find it", got) + } + if !slices.Contains(got, "Test Sheet 2") { + t.Errorf("expected to find Test Sheet in %v but didn't find it", got) + } + if slices.Contains(got, "Test Sheet 3") { + t.Errorf("expected to find Test Sheet in %v but didn't find it", got) + } +} diff --git a/cmd/dbasik-api/handlers.go b/cmd/dbasik-api/handlers.go new file mode 100644 index 0000000..c79de7e --- /dev/null +++ b/cmd/dbasik-api/handlers.go @@ -0,0 +1,256 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "github.com/tealeg/xlsx/v3" +) + +func (app *application) createReturnHandler(w http.ResponseWriter, r *http.Request) { + // Parse the multipart form + err := r.ParseMultipartForm(10 << 20) // 10Mb max + if err != nil { + app.serverErrorResponse(w, r, err) + } + + // Get form values + dmName := r.FormValue("name") + dmDesc := r.FormValue("description") + + // Get the return file and save it to a file + returnFile, handler, err := r.FormFile("returnfile") + app.logger.Info("got excel file") + if err != nil { + http.Error(w, "Missing file", http.StatusBadRequest) + return + } + defer returnFile.Close() + dst, err := os.Create(handler.Filename) + if err != nil { + http.Error(w, "Cannot create new file object from uplaoded file.", http.StatusInternalServerError) + return + } + defer dst.Close() + if _, err := io.Copy(dst, returnFile); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Get the uploaded csv file and name + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, "Missing file", http.StatusBadRequest) + return + } + defer file.Close() + + // parse the csv + reader := csv.NewReader(file) + var dmls []datamapLine + + for { + line, err := reader.Read() + if err != nil { + if err.Error() == "EOF" { + break // end of file + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(line) != 4 { + http.Error(w, "Invalid CSV Format", http.StatusBadRequest) + return + } + + dmls = append(dmls, datamapLine{ + Key: line[0], + Sheet: line[1], + DataType: line[2], + Cellref: line[3], + }) + } + dm := datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} + + // Parse the XLSX file based on the datamap... + // open an existing file + wb, err := xlsx.OpenFile(dst.Name()) + if err != nil { + http.Error(w, "Cannot open Excel file", http.StatusBadRequest) + return + } + // wb now contains a reference to the workbook + // show all the sheets in the workbook + fmt.Println("Sheets in this file:") + for i, sh := range wb.Sheets { + fmt.Println(i, sh.Name) + } + fmt.Println("----") + + //Here is where we parse our files. + + err = app.writeJSONPretty(w, http.StatusOK, envelope{"datamap": dm}, nil) + if err != nil { + app.logger.Debug("writing out csv", "err", err) + app.serverErrorResponse(w, r, err) + } + + fmt.Fprintf(w, "file successfully uploaded") + +} + +func (app *application) createDatamapHandler(w http.ResponseWriter, r *http.Request) { + // Parse the multipart form + err := r.ParseMultipartForm(10 << 20) // 10Mb max + if err != nil { + app.serverErrorResponse(w, r, err) + } + + // Get form values + dmName := r.FormValue("name") + app.logger.Info("obtain value from form", "name", dmName) + dmDesc := r.FormValue("description") + app.logger.Info("obtain value from form", "description", dmDesc) + + // Get the uploaded file and name + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, "Missing file", http.StatusBadRequest) + return + } + defer file.Close() + + // parse the csv + reader := csv.NewReader(file) + var dmls []datamapLine + var dm datamap + + for { + line, err := reader.Read() + if err != nil { + if err.Error() == "EOF" { + break // end of file + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(line) != 4 { + http.Error(w, "Invalid CSV Format", http.StatusBadRequest) + return + } + + dmls = append(dmls, datamapLine{ + Key: line[0], + Sheet: line[1], + DataType: line[2], + Cellref: line[3], + }) + } + dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} + + err = app.writeJSONPretty(w, http.StatusOK, envelope{"datamap": dm}, nil) + if err != nil { + app.logger.Debug("writing out csv", "err", err) + app.serverErrorResponse(w, r, err) + } + + fmt.Fprintf(w, "file successfully uploaded") +} + +func (app *application) saveDatamapHandler(w http.ResponseWriter, r *http.Request) { + // Parse the multipart form + err := r.ParseMultipartForm(10 << 20) // 10Mb max + if err != nil { + app.serverErrorResponse(w, r, err) + } + // Get form values + dmName := r.FormValue("name") + app.logger.Info("obtain value from form", "name", dmName) + dmDesc := r.FormValue("description") + app.logger.Info("obtain value from form", "description", dmDesc) + + // Get the uploaded file and name + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, "Missing file", http.StatusBadRequest) + return + } + defer file.Close() + + // parse the csv + reader := csv.NewReader(file) + var dmls []datamapLine + var dm datamap + + for { + line, err := reader.Read() + if err != nil { + if err.Error() == "EOF" { + break // end of file + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(line) != 4 { + http.Error(w, "Invalid CSV Format", http.StatusBadRequest) + return + } + + dmls = append(dmls, datamapLine{ + ID: 0, + Key: line[0], + Sheet: line[1], + DataType: line[2], + Cellref: line[3], + }) + + } + dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} + + // save to the database + _, err = app.models.DatamapLines.Insert(dm, dmls) + if err != nil { + http.Error(w, "Cannot save to database", http.StatusBadRequest) + return + } + +} + +func (app *application) getJSONForDatamap(w http.ResponseWriter, r *http.Request) { + // Get the DM out of the database + // dm = datamap{Name: dmName, Description: dmDesc, Created: time.Now(), DMLs: dmls} + + // err = app.writeJSONPretty(w, http.StatusOK, envelope{"datamap": dm}, nil) + // if err != nil { + // app.logger.Debug("writing out csv", "err", err) + // app.serverErrorResponse(w, r, err) + // } + + // fmt.Fprintf(w, "file successfully uploaded") +} + +func (app *application) showDatamapHandler(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + app.logger.Info("the id requested", "id", id) + id_int, err := strconv.ParseInt(id, 10, 64) + if err != nil || id_int < 1 { + app.notFoundResponse(w, r) + } + fmt.Fprintf(w, "show the details for datamap %d\n", id_int) +} + +func (app *application) createDatamapLine(w http.ResponseWriter, r *http.Request) { + var input datamapLine + err := json.NewDecoder(r.Body).Decode(&input) + if err != nil { + app.errorResponse(w, r, http.StatusBadRequest, err.Error()) + return + } + fmt.Fprintf(w, "%v\n", input) +} diff --git a/cmd/dbasik-api/routes.go b/cmd/dbasik-api/routes.go index 3c9e85b..1b02028 100644 --- a/cmd/dbasik-api/routes.go +++ b/cmd/dbasik-api/routes.go @@ -25,6 +25,7 @@ func (app *application) routes() *http.ServeMux { mux.HandleFunc("GET /v1/healthcheck", app.healthcheckHandler) mux.HandleFunc("GET /v1/getdatamap/{id}", app.getJSONForDatamap) // TODO: not yet implemented + mux.HandleFunc("POST /v1/return", app.createReturnHandler) mux.HandleFunc("POST /v1/datamapsave", app.saveDatamapHandler) mux.HandleFunc("POST /v1/datamap", app.createDatamapHandler) mux.HandleFunc("POST /v1/datamapline", app.createDatamapLine) diff --git a/datamap.csv b/datamap.csv new file mode 100644 index 0000000..9a9e13c --- /dev/null +++ b/datamap.csv @@ -0,0 +1,3 @@ +Project/Programme Name,rtp_template,TEXT,E6 +Acronym,rtp_template,TEXT,E12 + @@ -3,6 +3,20 @@ module git.yulqen.org/go/dbasik-go go 1.22.1 require ( - github.com/joho/godotenv v1.5.1 // indirect - github.com/lib/pq v1.10.9 // indirect + github.com/joho/godotenv v1.5.1 + github.com/lib/pq v1.10.9 + github.com/tealeg/xlsx/v3 v3.3.6 +) + +require ( + github.com/frankban/quicktest v1.14.6 // indirect + github.com/google/btree v1.0.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/peterbourgon/diskv/v3 v3.0.1 // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect + golang.org/x/text v0.3.8 // indirect ) @@ -1,4 +1,36 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU= +github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug= +github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa h1:2cO3RojjYl3hVTbEvJVqrMaFmORhL6O06qdW42toftk= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa/go.mod h1:Yjr3bdWaVWyME1kha7X0jsz3k2DgXNa1Pj3XGyUAbx8= +github.com/tealeg/xlsx/v3 v3.3.6 h1:b0SPORnNa8BDbFEujljp2IpTDVse3D+Ad5IaMz7KUL8= +github.com/tealeg/xlsx/v3 v3.3.6/go.mod h1:KV4FTFtvGy0TBlOivJLZu/YNZk6e0Qtk7eOSglWksuA= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |