mirror of https://github.com/gophish/gophish
450 lines
19 KiB
HTML
450 lines
19 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en-us">
|
|
|
|
<head>
|
|
|
|
<meta charset="utf-8" />
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
|
|
|
|
|
|
|
<meta name="twitter:card" content="summary"/>
|
|
|
|
|
|
|
|
|
|
<meta name="twitter:title" content="Handling Database Migrations in Go"/>
|
|
<meta name="twitter:description" content=""/>
|
|
<meta name="twitter:site" content="@"/>
|
|
|
|
|
|
|
|
<meta property="og:title" content="Handling Database Migrations in Go · Gophish - Blog" />
|
|
<meta property="og:site_name" content="Gophish - Blog" />
|
|
<meta property="og:url" content="https://getgophish.com/blog/post/database-migrations-in-go/" />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<meta property="og:description" content="" />
|
|
<meta property="og:type" content="article" />
|
|
<meta property="article:published_time" content="2016-02-01T07:00:00-06:00" />
|
|
|
|
|
|
<meta property="article:tag" content="engineering" />
|
|
|
|
<meta property="article:tag" content="go" />
|
|
|
|
|
|
|
|
<title>Handling Database Migrations in Go · Gophish - Blog</title>
|
|
|
|
|
|
<meta name="description" content="&ldquo;I got my database schema correct on the first try.&rdquo;
|
|
-No one ever.
|
|
Like most big projects, gophish needed a way to automatically manage changes to " />
|
|
|
|
|
|
<meta name="HandheldFriendly" content="True" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
|
|
<link rel="shortcut icon" href="/blog/images/favicon.ico">
|
|
<link rel="apple-touch-icon" href="/blog/images/apple-touch-icon.png" />
|
|
|
|
<link rel="stylesheet" type="text/css" href="/blog/css/screen.css" />
|
|
<link rel="stylesheet" type="text/css" href="/blog/css/nav.css" />
|
|
<link href="https://fonts.googleapis.com/css?family=Lora:400,700" rel="stylesheet">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link href="/blog/index.xml" rel="alternate" type="application/rss+xml" title="Gophish - Blog" />
|
|
|
|
|
|
|
|
<meta name="generator" content="Hugo 0.40.3" />
|
|
|
|
<link rel="canonical" href="https://getgophish.com/blog/post/database-migrations-in-go/" />
|
|
|
|
|
|
|
|
|
|
<script type="application/ld+json">
|
|
{
|
|
"@context": "https://schema.org",
|
|
"@type": "Article",
|
|
"publisher": {
|
|
"@type": "Organization",
|
|
"name": ,
|
|
"logo": https://getgophish.com/blogimages/gophish_purple_logo.png
|
|
},
|
|
"author": {
|
|
"@type": "Person",
|
|
"name": ,
|
|
|
|
"image": {
|
|
"@type": "ImageObject",
|
|
"url": https://getgophish.com/blogimages/gophish_purple_logo.png,
|
|
"width": 250,
|
|
"height": 250
|
|
},
|
|
|
|
"url": ,
|
|
"sameAs": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
]
|
|
},
|
|
"headline": Handling Database Migrations in Go,
|
|
"name": Handling Database Migrations in Go,
|
|
"wordCount": 1077,
|
|
"timeRequired": "PT6M",
|
|
"inLanguage": {
|
|
"@type": "Language",
|
|
"alternateName": en
|
|
},
|
|
"url": https://getgophish.com/blog/post/database-migrations-in-go/,
|
|
"datePublished": 2016-02-01T07:00Z,
|
|
"dateModified": 2016-02-01T07:00Z,
|
|
|
|
"keywords": engineering, go,
|
|
"description": ,
|
|
"mainEntityOfPage": {
|
|
"@type": "WebPage",
|
|
"@id": https://getgophish.com/blog/post/database-migrations-in-go/
|
|
}
|
|
}
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script>
|
|
(function (i, s, o, g, r, a, m) {
|
|
i['GoogleAnalyticsObject'] = r;
|
|
i[r] = i[r] || function () {
|
|
(i[r].q = i[r].q || []).push(arguments)
|
|
}, i[r].l = 1 * new Date();
|
|
a = s.createElement(o),
|
|
m = s.getElementsByTagName(o)[0];
|
|
a.async = 1;
|
|
a.src = g;
|
|
m.parentNode.insertBefore(a, m)
|
|
})(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga');
|
|
|
|
ga('create', 'UA-47018345-1', 'auto');
|
|
ga('send', 'pageview');
|
|
</script>
|
|
|
|
|
|
|
|
</head>
|
|
|
|
<body class="nav-closed">
|
|
|
|
<div class="nav">
|
|
<h3 class="nav-title">Menu</h3>
|
|
<a href="#" class="nav-close">
|
|
<span class="hidden">Close</span>
|
|
</a>
|
|
<ul>
|
|
|
|
|
|
|
|
|
|
<li class="nav-opened" role="presentation">
|
|
<a href="https://getgophish.com/">Home</a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-opened" role="presentation">
|
|
<a href="https://getgophish.com/documentation">Documentation</a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-opened" role="presentation">
|
|
<a href="https://github.com/gophish/gophish/issues">Support</a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-opened" role="presentation">
|
|
<a href="/blog/blog">Blog</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
|
|
|
|
<a class="subscribe-button icon-feed" href="/blog/index.xml">Subscribe</a>
|
|
|
|
</div>
|
|
<span class="nav-cover"></span>
|
|
|
|
|
|
<div class="site-wrapper">
|
|
|
|
|
|
<header class="main-header post-head no-cover">
|
|
<nav class="main-nav clearfix">
|
|
|
|
|
|
|
|
<a class="blog-logo" href="https://getgophish.com/blog"><img src="/blog/images/gophish_purple_logo.png" alt="Home" /></a>
|
|
|
|
|
|
<a class="menu-button" href="#"><span class="burger">☰</span><span class="word">Menu</span></a>
|
|
|
|
</nav>
|
|
</header>
|
|
|
|
|
|
|
|
<main class="content" role="main">
|
|
|
|
|
|
|
|
|
|
<article class="post post">
|
|
|
|
<header class="post-header">
|
|
<h1 class="post-title">Handling Database Migrations in Go</h1>
|
|
<small></small>
|
|
|
|
<section class="post-meta">
|
|
|
|
<time class="post-date" datetime="2016-02-01T07:00:00-06:00">
|
|
Feb 1, 2016
|
|
</time>
|
|
|
|
|
|
<span class="post-tag small"><a href="https://getgophish.com/blog/tags/engineering/">#engineering</a></span>
|
|
|
|
<span class="post-tag small"><a href="https://getgophish.com/blog/tags/go/">#go</a></span>
|
|
|
|
</section>
|
|
</header>
|
|
|
|
<section class="post-content">
|
|
|
|
|
|
<blockquote>
|
|
<p>“I got my database schema correct on the first try.”</p>
|
|
|
|
<p>-No one ever.</p>
|
|
</blockquote>
|
|
|
|
<p>Like most big projects, gophish needed a way to automatically manage changes to our database schema. As new features were being added, we found ourselves in a situation that required us to add or modify columns and tables to store the new data.</p>
|
|
|
|
<p>In a hosted environment, this is no problem since we control the database and can make schema changes as we see fit. Gophish is different, in that it is software intentionally designed to run on the client’s machine. This means that as we rollout updates to gophish’s backend database, we need a way to easily update (or rollback!) changes to the database structure. A versioning system is a perfect fit, which introduces the idea of migrations.</p>
|
|
|
|
<h4 id="what-is-a-migration">What is a <em>Migration</em>?</h4>
|
|
|
|
<p>A migration is nothing more than a set of SQL commands to make changes to a database. Every migration typically has two parts: how to apply the changes you want, and how to roll them back.</p>
|
|
|
|
<p>To version control our database, we can create a folder holding multiple migration files. Each file will have an identifier so we know which migration should be applied and in which order. Then, we can store which version our database is currently at in the database itself so if we ever add migrations in the future, we can tell where we left off.</p>
|
|
|
|
<p>There are tools that can automate this process for us. We settled on a well-known database migration tool called <a href="https://bitbucket.org/liamstask/goose/"><code>goose</code></a>.</p>
|
|
|
|
<h3 id="introduction-to-goose">Introduction to <code>goose</code></h3>
|
|
|
|
<p>We chose to go with <a href="https://bitbucket.org/liamstask/goose/"><code>goose</code></a> since it seemed like a mature, fully-featured solution that would be easily integrated into our code. Goose typically works through the use of its command line tool aptly named <code>goose</code>.</p>
|
|
|
|
<p>To set things up, we first need to create the following folder structure:</p>
|
|
|
|
<pre><code>| db/
|
|
| | migrations/
|
|
| `-dbconf.yml
|
|
</code></pre>
|
|
|
|
<p>Our migrations will be stored in the <code>migrations</code> folder as a series of SQL files. Before we can create migrations, we have to specify the configuration for <code>goose</code> to use. This is found in the <code>dbconf.yml</code> file. In our case, we used the following configuration:</p>
|
|
|
|
<pre><code>production:
|
|
driver: sqlite3
|
|
open: gophish.db
|
|
dialect: sqlite3
|
|
import: github.com/mattn/go-sqlite3
|
|
</code></pre>
|
|
|
|
<p>This configuration specifies a single environment, <code>production</code>, that manages a SQLite database.</p>
|
|
|
|
<p>Now that we have created our configuration file, we can start making our migrations. Unfortunately, this is where the hurdles began.</p>
|
|
|
|
<h3 id="a-little-about-gophish">A Little About Gophish</h3>
|
|
|
|
<p>Normally, migrations are something that is considered early on in the database creation process. Unfortunately, our schema was already defined and we had clients already running gophish. So, we needed to orchestrate <code>goose</code> in such a way that we could create and apply our migrations without messing up any data that was already in the client’s databases.</p>
|
|
|
|
<p>The first step was creating the migrations. To handle this, we first created an empty migration file using the following:</p>
|
|
|
|
<pre><code>goose -env production create 0.1.2_browser_post sql
|
|
goose: created ~\go\src\github.com\gophish\gophish\db\migrations\20160130184410_0.1.2_browser_post.sql
|
|
</code></pre>
|
|
|
|
<p>This command created a new empty SQL file in our migrations folder that looks like this:</p>
|
|
|
|
<pre><code>-- +goose Up
|
|
-- SQL in section 'Up' is executed when this migration is applied
|
|
|
|
|
|
-- +goose Down
|
|
-- SQL section 'Down' is executed when this migration is rolled back
|
|
</code></pre>
|
|
|
|
<p>For our first migration, we decided to baseline our schema to the current version. To do this, we simply exported our existing schema using the sqlite3 tool. That gave us all of our <code>CREATE TABLE</code> statements that setup our tables and default data. We then copy/pasted those statements below the <code>-- +goose Up</code> section of the migrations.</p>
|
|
|
|
<p>The one change we made was to add <code>IF NOT EXISTS</code> to all of our table creation statements. This meant that if the client already had a database setup, this migration would be applied, but no changes would be made - exactly what we want.</p>
|
|
|
|
<p>The final step to create this migration was to add the rollback statements. Since this was creating the database, <code>DROP TABLE</code> equivalent statements worked just fine. You can see our final migration file <a href="https://raw.githubusercontent.com/gophish/gophish/master/db/migrations/20160118194630_init.sql">here</a>.</p>
|
|
|
|
<p>Now for the next hurdle. Traditionally, migrations work by creating a new migration file and running <code>goose up</code>. Then, <code>goose</code> will compare your database version with the migration files it finds. If there are migrations that need to be applied, it will apply them in order until you are at the current version.</p>
|
|
|
|
<p>While the <code>goose up</code> command can work if we control the database, there’s simply no way that we can expect our users to install <code>goose</code> and run <code>goose up</code> every time we want to make a database change. Our goal has always been to make the lives of our users easier, so this simply wouldn’t work. This meant that we needed to handle the migrations in our code.</p>
|
|
|
|
<p>Fortunately for us, the <code>goose</code> CLI wraps a rich library that we can use. We were able to integrate this directly into our <code>Setup()</code> function to apply migrations automatically.</p>
|
|
|
|
<p>First, we created the <code>gooose.DBConf</code> struct to hold the configuration (a programmatic copy of our <code>dbconf.yml</code> file).</p>
|
|
<div class="highlight"><pre style="background-color:#fff;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-golang" data-lang="golang"><span style="color:#888;font-style:italic">// Setup the goose configuration
|
|
</span><span style="color:#888;font-style:italic"></span>migrateConf <span style="color:#666">:=</span> <span style="color:#666">&</span>goose<span style="color:#888">.</span>DBConf<span style="color:#888">{</span>
|
|
MigrationsDir<span style="color:#888">:</span> config<span style="color:#888">.</span>Conf<span style="color:#888">.</span>MigrationsPath<span style="color:#888">,</span>
|
|
Env<span style="color:#888">:</span> <span style="color:#b83838">"production"</span><span style="color:#888">,</span>
|
|
Driver<span style="color:#888">:</span> goose<span style="color:#888">.</span>DBDriver<span style="color:#888">{</span>
|
|
Name<span style="color:#888">:</span> <span style="color:#b83838">"sqlite3"</span><span style="color:#888">,</span>
|
|
OpenStr<span style="color:#888">:</span> config<span style="color:#888">.</span>Conf<span style="color:#888">.</span>DBPath<span style="color:#888">,</span>
|
|
Import<span style="color:#888">:</span> <span style="color:#b83838">"github.com/mattn/go-sqlite3"</span><span style="color:#888">,</span>
|
|
Dialect<span style="color:#888">:</span> <span style="color:#666">&</span>goose<span style="color:#888">.</span>Sqlite3Dialect<span style="color:#888">{},</span>
|
|
<span style="color:#888">},</span>
|
|
<span style="color:#888">}</span></code></pre></div>
|
|
<p>Next, we need to figure out the latest database version supported by our migrations. This gives us the final “goal” migration that we want to upgrade to. We can do this via the function <a href="https://godoc.org/bitbucket.org/liamstask/goose/lib/goose#GetMostRecentDBVersion"><code>goose.GetMostRecentDBVersion</code></a>.</p>
|
|
<div class="highlight"><pre style="background-color:#fff;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-golang" data-lang="golang"><span style="color:#888;font-style:italic">// Get the latest possible migration
|
|
</span><span style="color:#888;font-style:italic"></span>latest<span style="color:#888">,</span> err <span style="color:#666">:=</span> goose<span style="color:#888">.</span>GetMostRecentDBVersion<span style="color:#888">(</span>migrateConf<span style="color:#888">.</span>MigrationsDir<span style="color:#888">)</span>
|
|
<span style="color:#2838b0">if</span> err <span style="color:#666">!=</span> <span style="color:#444;font-style:italic">nil</span> <span style="color:#888">{</span>
|
|
Logger<span style="color:#888">.</span>Println<span style="color:#888">(</span>err<span style="color:#888">)</span>
|
|
<span style="color:#2838b0">return</span> err
|
|
<span style="color:#888">}</span></code></pre></div>
|
|
<p>And finally, we need to apply our migrations. <code>Goose</code> has a function called <a href="https://godoc.org/bitbucket.org/liamstask/goose/lib/goose#RunMigrationsOnDb"><code>goose.RunMigrationsOnDb</code></a> which expects an existing <a href="https://golang.org/pkg/database/sql/#DB"><code>sql.DB</code></a> object. Since gophish uses the ORM <a href="https://github.com/jinzhu/gorm"><code>gorm</code></a>, we already had a <code>sql.DB</code> object already initialized that we could use to send to <code>goose</code>. This was stored in the <code>db</code> variable.</p>
|
|
<div class="highlight"><pre style="background-color:#fff;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-golang" data-lang="golang"><span style="color:#888;font-style:italic">// Migrate up to the latest version
|
|
</span><span style="color:#888;font-style:italic"></span>err <span style="color:#888">=</span> goose<span style="color:#888">.</span>RunMigrationsOnDb<span style="color:#888">(</span>migrateConf<span style="color:#888">,</span> migrateConf<span style="color:#888">.</span>MigrationsDir<span style="color:#888">,</span> latest<span style="color:#888">,</span> db<span style="color:#888">.</span>DB<span style="color:#888">())</span>
|
|
<span style="color:#2838b0">if</span> err <span style="color:#666">!=</span> <span style="color:#444;font-style:italic">nil</span> <span style="color:#888">{</span>
|
|
Logger<span style="color:#888">.</span>Println<span style="color:#888">(</span>err<span style="color:#888">)</span>
|
|
<span style="color:#2838b0">return</span> err
|
|
<span style="color:#888">}</span></code></pre></div>
|
|
<p>That’s it! You can find our full <code>Setup()</code> function <a href="https://github.com/gophish/gophish/blob/master/models/models.go#L61">here.</a> To handle any additional migrations, all we need to do is run <code>goose create</code> again, add the SQL that makes up the migration, and push out the new file. The next time clients update gophish and restart the executable, the database migrations will be applied automatically!</p>
|
|
|
|
<p>If this kind of stuff is interesting to you, and you want to see a full example of a web app written in Go, check out gophish by clicking below.</p>
|
|
|
|
<a href="https://github.com/gophish/gophish" class="btn">Download gophish</a>
|
|
|
|
|
|
</section>
|
|
|
|
|
|
<footer class="post-footer">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<figure class="author-image">
|
|
<a class="img" href="https://getgophish.com/blog" style="background-image: url(/blog/images/jordan-wright.jpg)"><span class="hidden">Jordan Wright's Picture</span></a>
|
|
</figure>
|
|
|
|
|
|
<section class="author">
|
|
<h4><a href="https://getgophish.com/blog">Jordan Wright</a></h4>
|
|
|
|
<p>Read <a href="https://getgophish.com/blog">more posts</a> by this author.</p>
|
|
|
|
<div class="author-meta">
|
|
|
|
|
|
</div>
|
|
</section>
|
|
|
|
|
|
|
|
|
|
|
|
<section class="share">
|
|
<h4>Share this post</h4>
|
|
<a class="icon-twitter" style="font-size: 1.4em" href="https://twitter.com/share?text=Handling%20Database%20Migrations%20in%20Go - Gophish%20-%20Blog&url=https%3a%2f%2fgetgophish.com%2fblog%2fpost%2fdatabase-migrations-in-go%2f"
|
|
onclick="window.open(this.href, 'twitter-share', 'width=550,height=235');return false;">
|
|
<span class="hidden">Twitter</span>
|
|
</a>
|
|
<a class="icon-facebook" style="font-size: 1.4em" href="https://www.facebook.com/sharer/sharer.php?u=https%3a%2f%2fgetgophish.com%2fblog%2fpost%2fdatabase-migrations-in-go%2f"
|
|
onclick="window.open(this.href, 'facebook-share','width=580,height=296');return false;">
|
|
<span class="hidden">Facebook</span>
|
|
</a>
|
|
<a class="icon-google-plus" style="font-size: 1.4em" href="https://plus.google.com/share?url=https%3a%2f%2fgetgophish.com%2fblog%2fpost%2fdatabase-migrations-in-go%2f"
|
|
onclick="window.open(this.href, 'google-plus-share', 'width=490,height=530');return false;">
|
|
<span class="hidden">Google+</span>
|
|
</a>
|
|
</section>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</footer>
|
|
</article>
|
|
|
|
</main>
|
|
|
|
|
|
<aside class="read-next">
|
|
|
|
<a class="read-next-story" style="no-cover" href="/blog/post/2018-12-02-building-web-servers-in-go/">
|
|
<section class="post">
|
|
<h2>Building Web Servers in Go</h2>
|
|
|
|
</section>
|
|
</a>
|
|
|
|
|
|
<a class="read-next-story prev" style="no-cover" href="/blog/post/release-0.1.1/">
|
|
<section class="post">
|
|
<h2>Announcing gophish v0.1.1</h2>
|
|
</section>
|
|
</a>
|
|
|
|
</aside>
|
|
|
|
|
|
|
|
<footer class="site-footer clearfix">
|
|
<section class="copyright"><a href="">Gophish - Blog</a> </section>
|
|
|
|
<section class="poweredby">Proudly generated by <a class="icon-hugo" href="http://gohugo.io">HUGO</a>, with <a class="icon-theme" href="https://github.com/vjeantet/hugo-theme-casper">Casper</a> theme</section>
|
|
|
|
</footer>
|
|
</div>
|
|
<script type="text/javascript" src="/blog/js/jquery.js"></script>
|
|
<script type="text/javascript" src="/blog/js/jquery.fitvids.js"></script>
|
|
<script type="text/javascript" src="/blog/js/index.js"></script>
|
|
|
|
</body>
|
|
</html>
|
|
|