From 9279f831535b68879ba06795b60876885264885f Mon Sep 17 00:00:00 2001 From: oabrivard Date: Wed, 18 Mar 2026 11:29:19 +0100 Subject: [PATCH] Finished D2 tasks - VNet design & deployment --- .../Terraform/environments/prod/providers.tf | 8 +- .../environments/prod/terraform.tfvars | 11 +- .../Terraform/environments/prod/tfplan | Bin 0 -> 12351 bytes .../Terraform/modules/networking/main.tf | 5 + Implementation/steps-d2-d5.md | 436 ++++++++++++++++++ 5 files changed, 451 insertions(+), 9 deletions(-) create mode 100644 Implementation/Terraform/environments/prod/tfplan create mode 100644 Implementation/steps-d2-d5.md diff --git a/Implementation/Terraform/environments/prod/providers.tf b/Implementation/Terraform/environments/prod/providers.tf index d7cbb0f..62a0b19 100644 --- a/Implementation/Terraform/environments/prod/providers.tf +++ b/Implementation/Terraform/environments/prod/providers.tf @@ -39,7 +39,7 @@ provider "databricks" { } # Workspace-level Databricks provider (configured after workspace is created) -provider "databricks" { - alias = "workspace" - host = azurerm_databricks_workspace.this.workspace_url -} +# provider "databricks" { +# alias = "workspace" +# host = azurerm_databricks_workspace.this.workspace_url +# } diff --git a/Implementation/Terraform/environments/prod/terraform.tfvars b/Implementation/Terraform/environments/prod/terraform.tfvars index d67ed72..9e90f23 100644 --- a/Implementation/Terraform/environments/prod/terraform.tfvars +++ b/Implementation/Terraform/environments/prod/terraform.tfvars @@ -27,9 +27,10 @@ adls_replication_type = "GRS" # Tags tags = { - project = "mdp" - environment = "prod" - cost-center = "Greenfield-CDO" - owner = "data-office" - managed-by = "terraform" + project = "mdp" + environment = "prod" + cost-center = "Greenfield-CDO" + owner = "data-office" + managed-by = "terraform" + data-classification = "internal" } diff --git a/Implementation/Terraform/environments/prod/tfplan b/Implementation/Terraform/environments/prod/tfplan new file mode 100644 index 0000000000000000000000000000000000000000..8b7260776c0e17a7556d206fcb89d1e2b639c5e6 GIT binary patch literal 12351 zcma)ib8u(N)^;YgZQGgHoY>~X6Wf|N`NeiJv2EL)U}D=gzMS{gy=Ts;dry69*Z!kx z*M4es_v)pm<)uJCQGp;o9+9tB+CaYvI1m(&qlvAhzO{-n7|^Gxvs$wgZ4XP_LAU*w z_;K0rzBN*?9=0L}vow(@ymMF7+mN`#L{u$n?9$jJVAk;Nm9rJQzV?*cF02}QO#5pkQr!6DqCd2peYxCp{LL(kJ!w94#+6v3NSr)Y_Hnc z(8W+ZKm4+D#$WgBTz}bHwsCg(CV&HW9*?UaNQ&y5VgTi2V0%i@<&o*-MoB|TQJ1Pi zjegZH#H2TbAtp9hpRutz#0a*tz#JM$s*gIN0b_yi-K9@6!)@n)>6Gch8bm^QPb3Q& z@sO*wIG}F|@Rd~6h1cU|Dke$}mm~gpu6K}N950=N@A6{07s7i}-FmCbchKh%WzZoQ z!5r*D;Ons?veaOm&&YSd&=R5NXV{0vikdP{;A7_22~skjTl$?et=-#Jji+-S>4d(AhZzKLU&Ma=+v9 zt)@Ff*UJ9k&#{;W=IY_{RBXngM0!`AI>6`5BNUsYC}~Q?(G2bpd_Y0cdkRI>Igymi z(O1}C3hs0-8YEgzHf3Q?Pd7T$S0^nzx;wmY=Mq(L^K+NEv%Y@fBbw6L1ZRZ=JlY z8U`NNsLgkgTPuz+G1ojxIT<$?Tb}4!pNSL?z%KW>0d`@xl>$_+`G}(pyDcjW)py4l zu3`)uu{OoyJRsLq14stpD+@&)q@XwFVAao1H%sjTyp%jH$iFv6W9oJeuo=>jZWjSO z7S>cPy(;0~g(WR3X)N`IzL#bgv?32WLr=<*EXVWOqI}nqhPB?9w_0;4wZwJ+6s>8s zNnFTuoQp-`d3!P9_3$4MjC^}vLwKuyci|1FoSTvmKh?x?7^03g%&-H2rtKf^=(xTA zrI$gjy)+ciQ9MQJI8xroEy&9uyhC~;<8;}y_-nsrU-UgvTIG5VUlenMuC)B5Rr8#1 zfm%Qv74}hPE?H}AWRlI;82SwDo^4lG*pB;bP9-CaySkP2-mlfjAKDe5fE)Gn#!dG8 zCR}|!=Y7Gj+SgIpw`jWPnYw{B3FDSA=f2Xaw1RW`-Gh&|_m@i^$=W`65c5(N?00_r zrn&nrWMbI0Ny1V@DAZ|jNr4k&EJ=5du64!Ca9mr!xbII*Z97=~aXwS(2?xM4 z(S_Q@oJEh+31i`H#OC( z?+upX`{gU#sDufRk2JBw4ct#vLvY<^w5X=6i!p9#3ML9<9E)*BMC9IP14iRgDada* z%xoF;aH8@DV?DCc6qBl%%-<+|%O7}}FwM3dT~j$8Fjdk5>Wgq9gA2*rT9L7EbKoG2h=y zLU@mJHk&pv(mWg8wjLYk_}qUFV$~B2w810F>{RunC+XDr3*oIi&2R1&Vtcwo0m&`H zE#ZAFStdSEmIYQBxM`;jep$slSSjbbb}L4m zHBH!Z9tw`XDnZpO^wR4vIlImgb&*T&`phKvP#aqzD$dQ!sNp-KG5)g+i!jz{CG!RY zbLTeFr05&Kk7vIS*`8WN#|jt2IulVdG!EL}$b3#p4V6=RSxG0~s1}cLU^fKCw`oP( z%mscG_H-~#c-gTX+&bvYYtn*B{~ZVHnFi}(oNFOTdm-X#m@V2S(f7=E4fJMko${bM zcu#wUKaTn96My~WiyXge`TLB;k+loBDrT18N*8!ME>&||^^wlSd1ZEIefy1ZWz#fI z-m!WJ4t`kTT#z-moE8;NpihCB{+NFiIYo+8l|^`JZM6E#w!98`9ri`P#=C|}4ltV( zXQ!Pql5(3}!ZvnXllPE?x4u#)sj*I9BiOJ+J)qgPVhLEd(_x)|D?edCUlYP{b!3DVp|JY13%9zbE_R znG=8(E+Yn6fTBj#{(1hGYWJle_S(j@i^j>`n={^lpeQrP1=9R@&i;mt+THg0!h=fm zm6hyh{TyO_6Y95ZcUQpRPAi5Vcu28lwhU zmNe{+`2Y(a1-HB-;FmE9-jvWhW(>aRPFlt+a;~7uqy6Z#F_bKg_17?wed#K|*z-OM z$&Aw=t3e+M29v!X64w^1whyTrOPC>y-ANL`rINkim*6-TqnQRBx0K|fR{v}ku@!IH zJi}???SqDP`XgcmH245rZJ+fZSVKyf2#JAPZF#jAaPT*qG6IZC5ts}2DWz|IjX^|A z0S@_>Ikt&?r1p(y=Xtc6%}}Jzo5CAigLmm55hnSerGZ+yPBj{eLR7`_;PBHU@(Y!XP6*Wt!^9(wFsjW z8$`um39A$jfQZuxCrc|1v zF~I)G=lpEE#4YEUPGB3sT5_8(&YsR#)6!T|d=w&(Bxb>ft2`mX`Hazt9pQYqw~RZ25|pc}XlIPafEe%! zfI+8_PWj=@H0)@S$yuRQ+VOt0Z*L~_3H7#dzAiEKauQT+N$w7yN0&Nwm@$eo(&wpZ zbhLfI9rg1tBg3F>YhqvE2g~4MuJjOMyjllVj=j67gY?l%stX)(u1;i#-zEo)T`1zC zn1Kku1%o)BPKY!+mX7Cauv=IDSoSakBMBc`sL!ORPPKA|yA?l7%=c9B^Zfd{i6NA^ zb6xEU*a9tX>VPtzdD1x#rRhD|ol_*BQ4E-dz3XCxVjY(wo%1ff84NLAGVn5zsOe;4 zXF+KX{CtKz_<9oWxaOxK%*+qJQ!PdEi%?Gd8ZMRQeJs>12~_ ztA4$;^+vO<*;hV24r#JI)KPjnFOcIoWndWgs*X{_g`-BAvk`XhfhC^q!+o-2q9L5! zxZZWyl-v@ZlUvPeq!xaiTdB0d($vW^r}=X&8>^@Ww=a1tQL@ZLa-OD$E641=I#?k) z_*_fR3(;SNBdK_f%S>ase~8jrA_ICIATkt=I&mdnO*Pn|3XqSIr4$w112T?rMt=kb zB75I1!GoY&Ps|UThz7HJfgm;{k^gX1PL57m*Lc%kr zH~ru=Ex0&I2;ukMMT@JZTJ|%JOXx*GG%XGyQ-t1sntU{ni9Xn3CJBP)^d1=y5*jXG zbfW*2u(;xgzKVQ)BzePa;$5qKD9W#Pq~)SZxOrbqNLkZI8aDxW*J-)f9zLtCjTZl9 z5wzR5V%uh~YXIOW)T;#x`X(tLN*G53f()W0u~X)-*q&G92ODC^>9(CtJuS1<#Bc7$ z0do|b8pGmQ>Ek$XPPG?i_{dwx2g~JStk?*sBdzJ$Q_A|Ap9diKfR{Hc%+T9L|y z<8IRTa~66S*3|ri$)ryI5+5j+_bS7D3(8FH`S<|WgLn%(@GqyIJnkICmK}|Tv$?)X z(R8VImYlT8 zf$KoK)~}(JJHl|B)?3fPo6zR3j+o&WC-Up}Pe4HL@A6U*kd_fIZ(reofQk)( zfaL#YaSHK&#i@g%zN7Jfs#B40!IEA=RMCBnz_v?vQi8Z-%lvr4C(C@*Q6+AuMGpmN zyi5m#`vXjWc13%pu7N(i-Eff+G551humNw>S717Eaap&8g@vis??*xPHTI0ELkXpi zdiHeaqxmAEg~MfH_r2>boq3lLok3jvZ{Nz5!6su^BR z8j)1(k~Gj8xju;wbRm_KuM%FB4(@_1aZi4JOUWU3&8#y8K}HqI)Uy~Pm4EZY?$x9A zt-1nJe_038jAi)q?Cl;kKUJqH5$jzow!=a%!@m_y$wY3m{!ljN9~DObKZOl#tWC^J z8La3S4D<~xjIE969ZmkDdRdg9;zIQfe?U#oC?+NGxvanGB%jnxILlxwM{+pIW25_*};=GMI*YJGEhc zZ5itM*(~T+t|q`SNE|t+VIZ+BxMb&v01_BeZfd8DDM8ccWN6B`XDhNosXJ-uWU zjMTFRdsoE|68_EQiK)V8lfa}g!&?e|#p&RnQdG9C;qREtoqUgJ&qCQh(Q9mrNDI*}siW3}jlD_GPmwP=|;^BM_S5joy72Xt@ zl@T60KM#gTty;INSoxw#g6iXbZvlJ7#fEL?Y-IpaVD!~~l3rzHx%+YeG; zwa|##7FZGv?GSh|VvuW?>dsXw!Fk3`(r3ob&lY)l>0Ws>Ke5TC2ak(v57T$WKnEG$`TooZ1LNI5SXbP^!{ug9B&az7i*nZj(62samm%ixKxX; z++{b)y>e=@5ew%(Z-}Z4CtNofVx7U^Xe~(JOKwIAIs#h(0B=fCC8k9vlHetA=EM!n zyQRfih2g}UR=2zYZwAGuQDDtE(rdU^mf7}}U;$^euSaF1ZDTi%#c7u**pDcs+jcAcHv#G4M;n}rW39i!zHs);XTw3Lxw#Jd7t_a-kI20jzI;w&B=0z%Z>ni$>H|uj)1$Q(pSP?LstUiEL zRw7gu7X%I=u`YkBZd;m zxV5s|72#QU4QQ7l*6qgy zFP}N2j685|A0@B@qRzBIa}0;VK@xnxJJ`RyQ%i55wlXLX&?&}0c_+-jc_&+Y8)q{k zV|$0+T~t}zBmk8Oap2qykh7ly;h~KPmW|PeCfB3eXILatV%l>@YNe8I9FIiMLvbUK z_s(9^1i8mZT3lJup(o>h?~}CH?LiK3`Z3TOoTl|~yG^lTf3;{Pjm2MI%1Mv}6r z@Y`zfE4@;AY-7~wK9LLJ0eVSIc>tYr1Wy)tLjDqOXdAlxN=idMUNC-d;X2KJSjo^^ z`K6y5{!ACYqGjOz_-=DVZFf}WFsvD~7Pcvw7mn9lXk40Qo;SlPxOuu?Anj2=Rk3_t zNm!Ci^jIQ(28+6=8fi&j%%_OgcH~quusD|^0g`G-UMnfbAjDulJyVNwW}xX)v?oQR zFn57l(GOwz_oh=Qmy3bs_*_1VqFiw6c{lnCxud=dCUD+fEoIA}pJekWQ*XU?PzU}pr>c8?A15%Pe_~P1{4<)hU#&*P;Mc;?J=JA< z-NAuP!1XUN{3=z02WC@2L|f^i9Ri55AL__zE#}@2&V~-t3GT3p^k34NnJ})o|7{y9 zee~|*A9+{Mhkg9-^yKfh;jC|Orf*U^Mu@8=E) zoEM11;6(-O%Qt;R_cNm_H{D&-*DJ)U$18U83{})_XPhmE;!x!3ojZ2xaL1`~2b4Z# zh?X^h(=&5oBdhZ*2y3}bA*bATeN3~K6%EU`atIrPta<0X2j7)!7%Dw8v!_jiqrU}y zq_Znfv4Lqzk8HRFEqL~}j5(w5g5=ZC;qW!?&oF)*@&m1&In>0&?6bQ>a-f=GsHJ1v z^eQ&dtJ=!M+zr3_78fkBOr6x331m&FO3s${+Y!rBYZZ-cO9`7h5X$nN@7#HuhB`zx zgl9>`50zeS=VWa8b)&^X{+!gzJ?aIM#)rih{v)pgjOx>GB^|(82CU2c;&3qITdjo1 z)RgL|OQ_EaL5nl&ERKDHzNmNjlJWZwP1dmFW_dN@ zN;r}d^Tj$+LiCmP3WT9e^o$y9t-JPu}C*JkXS}7N7rHw^BY?A6(P;@|OO6`x9CVQ6DyWzi2 zofL#49BPTHY||t?YkgL?7NOZEz)khk?<{ngT+UTUQKBia#eDcU-D|F{p>UMmuhX|+ z7J;`#lgki%2tq*N2pO-9%B-@Mq9+=??10U4@~TFD?oHi~ETz zEjxj>%@so&R&a@H4aA#lmC{lM%m9O{+@s*AZ8qt&yFmt^jeRBl*T}$SYm#n*Cgd45 zk2RpG3MrXp2CzgjTW)=4R0FlimkZi=@P8jj5}3C|ArK%S0L(ufNW8x~Yinai7aMyE zGwY9B{BKF+iI$D^auf1#% zO&t2CJ4BD*_Cb7Oxf(*z;aKA3c)G#0Gw22WilE+l&jdp zi**e;gWrHQlsypVC@m{`SH;41b$18-B_6)nUprvx;L4Q}*3pl99h&qO%C0!my74nWUaePn z8r0w2ed9{udDsO`heGhQBV&XM=qQjhTnGWx_D~K&ASZ^AYZm8LiO5V}@;fQE?N(G^ z0xRbw9Tq9+_P2oYG6~~Gqmd99roxaX=lPw!Mx!arbr}U*Ng4;{O*$+V{dn<$k>CVQ z&Pyj)b1oUJ7Vn<^!-R^s>sZ?tN-}-Prf`J=c$rK&;5W|66%gS)(uQR#j&XCybQYp< z;Cx*bAhH`Nl_K!QH8_;r1qRs#GHI~3W%7*nc}8}O4kBveLyM?qSL zWdd6O?>kViYkYKRI-B+Vl?t~5tiV@3GQK{l@;|`0?yBGsaac$OnSK9`T_XlTr2)gs3N#uLJY6>)o4lix(!GYnCg|DG~Vym z6yB9ZPqf6yh*7Vx^^Xe;=fhVYA9&$6#)j&1nl`rhv)kI`JGP7fq(Lm$A|GTV?KX$x zfEx#1BWWT{mgpz^%|L~Afl2V}4k5$S2ZoZzc+b`N@6>z#Pq@_3M(AQeU%=+DMEwyV zw4U%=_ClbE$0zoE(iIl@as|D>5gYi$Bt=*ir>~! zuG@2*n*PKBDZj1=e|gJKldW|t&DVcDEDX=wT|a$Ct?_W0a8ocyegwTFy{09pfT+M; zR_$GiT@;<)i!oBG;LKzUCE$<(4jUrz1++36vDgefz+M7DK*w)JKP&+a8)|3Nhjq*J zDlywU7V08tvq*222A{&NC~8*tSMCjBuh^?+4T98ujUf_Amy!7HZ(7eG4)Q{dPx|mo zTVVg`t)yV3&p$6V4(M$TbnyuHMvSJ!#%scNqV2~@uG)IK`ipL@sNUE zhu&3z?r5iiE~281*rRb5&RY&a=ks~2W{WfYkh-e_a|jbdPxBPxskh_Px*|SKBSTMI z(L9@a+Dk~~WMeZCo@c2$Or8Bn;rj&3qy{|BM8t!J**gb^jM;)17}9gziC2nr!358A zQ|%c~ol+qK{LDmgW$pcywMj>lxwwd<9(#pPa#&m84EK8;)O9Ajx-`hDg920h(-2xu zGrq>{49LAmd8@oHfC$VhuPwnE}qUAZEPn!#sC9&=$etbL6=3wRS zilsk8Q;hgvO2H~$hi)E`8A%562a)hL{T8l8D7%D^Q_DoihVh#i^zbnp(K<2q8>oQg zU<#8&$MfwDZ|-pr9VY_XOl-o+$MfUc7O==BaY%NW!SKRlc~xs`>Q>u`^stnEo$zq0 zXmtOYUI$c)7HP$*3f`xBW*E4;AP3#`pBdlZmYNhRlAjPLsiY<*G#Cj!`Or2t66cT5 zsn=-50-$jui}1=MRFzd|wJr9-gL2+k%X$Z=5xNI2?e0|!18zElaN?p@v`U2g!U+6A z$We0nJ74OdFie%Ccini3gaNFlT1H8&5vGTte(@f0A?dmkEoWy5WYPTKJ|V4ZaAXE zVlxbH`sWf#xqZ}2hyfp&tI26-47>M(WsMhqtm3QhRP}_2ag%(#=DSmxXhQSJkw1(2 zt_)G5>KgKaZz=d8PR?j0_`=g$*ATM(?tO-*m|B)R1)A$^0#NBg70~$BktBX&f|}|b zX3L6I!uS-Zm`jKV~eKnmkJ;$wU`K^XO@3Q zb9+aZJ9h{mpcl@6ispp>Gn(5tIodio{x!``kf}&xL>#X5F7xX@nX0XX?B~5Wv z;g%Il$P^9LoJ2A=+4=(k;$Hi}`E;lhx{y$Nn9O|_KZ`->+?i8L;vKeWlD$BBiX<4bKhMFt|%Ii7wF~n9dAr z2@%f>WKF2J3WVd3uM|L>g7h*``Huzb-s{{?#=|#>l&)v7GbkghSIF4jua>;&Y)n$8% z@A+PDIMS@C>evg0Xr>OIPoz04b!zpPzp>z(tAra4xpbC;5!XJQu?m@Bao~pjen|0? z^MyjO9U~28#LknssrIFNuFWIU!Y~Ks5O<*=zWaRrt_wir- z$WY;%8p4lJ#6KpH#Q)Rm{(F5F*KIw^h!}M44N$nNNoX-sL~P_G1jr*O7X|B*P-Gg( z#8tCqf=bpSBXpx&kq(iDI7|3Oa258&dA`5I@t5w?)%(%J%SGDIv0kjDbcjB@-Qui# zYiO}#D)J$Uq|Fb4f|EF=rkhs)zK8DmTD0)w3Zx~j>vM?ZTOEo(3DkTIsbob2sbxO< zMg(tGby6f8tFg7b?nSstRL0Bg039UPy+Me%v-2L$(;B5dUh0YIkt=k{q1(5s8VOGN zn3gtF;`0q2LCVHV$rGA-^%c+wT<8h_H_ZnO{`4X_upSfpvi%%OiD7D^>fM zdsL4sDk9~|q1r(ZFqH(8?3S95(V4EV8Y!YBqzlQH1>(`*k4_sM?vhAK8_=2T4`TtX z3fNXld)GdV>mp;39qO&DKOVK?voOhZjp)ABCeqgZUb=}#qG(mkT6H|iB+K*zpXsOv zo14Lf5Wc7UR_-0zr|)K}*t#dXUiSI+;c1w}=iZ}UjsHIz%hfM;_x+<>pZmvh9qlh0 zYh&c}N2PA=VDslCKER%$e9_1IAy=~H$~B7eia&jAb(q5hw0X(s!2&h$V2$01$#x$_ z7gO908R5Dg^F@b?zw)|2xw}6#KT_p$7WY3TQ|pUuD}vX=(4Zs!+wxduiTPAMI^k*m z$R7OB38!~7wzt|p#NrQ`5)>aAYLcT&X=%L?xwq(L?-hJ2 zAFjo>`0B;So0o5etEZcr###)^r&I_KrTe|x4g9Bt%dKgX4GgD;mIzt{-YuUyEomBq z_qWZKvl_UvCW2Qc&UVN2*me zF7sPAoC@PdK3~r`yLBvBU&14uZ`_$IzdRnga%o3;wJE>dmcpIi434zponK#FPO>L8 zEPK4XRCd12`#hhy1V;_B7xG6Ez@L-;acb)cbABiWghBE{M?whUdph%UX zFTe%92bqZ>th}#nHU@<|25(0PP3;HHb_HCgY=b(^;;xrLrkUVyY@y#UOQok?*#wsQj*#W_&}m`yMUXgtJnF*RStlC4E2*M8nIT7Cs%n^3uu5ky-QP_!r#1e+fI6N$50 zHT8RJZK&RU?WMr{TE@(B7d;FuNmU?5-GM%nipRdSmuOrEJyPWexv_Ib0G|*ujRCn_ zB(2`K6DXW!Zh?8L$mlo}$c%h-p4+Z=*cYu+?A@-d{E{ww4%Q5zSZpECaU;k}T&N*f z9{*XbPmN!(&0W2^H7u^ux7;65I7*_zn8Q461+#<|_q4gx$mTw~3?F<37=rhpnX_UC zM(>dYvCY^U5EQpSJKe2W0NV|!lwQy7Fy9WH@HuSMtI_Dkb7zB=f%BN0lgH!P8rKMC zPtKMR)JAbqKihv)Q`YwF`f)jv(@K6G=+02xx`#8Wfw@%S3KHej{uX%+h|>pxN%0M z!8SXZ{FO`o``^Daqkj;#f66z6|NQ+|`u6V-zY~RjP^frY9>`3u}Xw88$4`Fn=+NBaGzn2`K0%>R;%{~h-C`20tD^{1?o{V&-6uQm8RuKp3; v|C9!b-+}*Iu>ZS~zaOu^+Kn0|&_Bjyc`2}uMkNps?8g)FaX`~){}}y0xvhF~ literal 0 HcmV?d00001 diff --git a/Implementation/Terraform/modules/networking/main.tf b/Implementation/Terraform/modules/networking/main.tf index 110ddf9..22612c4 100644 --- a/Implementation/Terraform/modules/networking/main.tf +++ b/Implementation/Terraform/modules/networking/main.tf @@ -380,6 +380,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "databricks_main" { resource_group_name = azurerm_resource_group.network.name private_dns_zone_name = azurerm_private_dns_zone.databricks.name virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags } resource "azurerm_private_dns_zone_virtual_network_link" "databricks_transit" { @@ -387,6 +388,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "databricks_transit" { resource_group_name = azurerm_resource_group.network.name private_dns_zone_name = azurerm_private_dns_zone.databricks.name virtual_network_id = azurerm_virtual_network.transit.id + tags = var.tags } resource "azurerm_private_dns_zone_virtual_network_link" "dfs_main" { @@ -394,6 +396,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "dfs_main" { resource_group_name = azurerm_resource_group.network.name private_dns_zone_name = azurerm_private_dns_zone.dfs.name virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags } resource "azurerm_private_dns_zone_virtual_network_link" "vault_main" { @@ -401,6 +404,7 @@ resource "azurerm_private_dns_zone_virtual_network_link" "vault_main" { resource_group_name = azurerm_resource_group.network.name private_dns_zone_name = azurerm_private_dns_zone.vault.name virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags } resource "azurerm_private_dns_zone_virtual_network_link" "purview_main" { @@ -408,4 +412,5 @@ resource "azurerm_private_dns_zone_virtual_network_link" "purview_main" { resource_group_name = azurerm_resource_group.network.name private_dns_zone_name = azurerm_private_dns_zone.purview.name virtual_network_id = azurerm_virtual_network.main.id + tags = var.tags } diff --git a/Implementation/steps-d2-d5.md b/Implementation/steps-d2-d5.md new file mode 100644 index 0000000..bddb957 --- /dev/null +++ b/Implementation/steps-d2-d5.md @@ -0,0 +1,436 @@ +# Phase 0 — Remaining Steps D2 to D5 + +**Status as of 2026-03-18:** D1 is complete. Networking module is deployed (VNets, subnets, NSGs, NAT Gateway, DNS zones, peerings). All 6 resource groups are provisioned. Terraform state backend is configured. + +This document covers **only the missing implementation steps** required to complete Week 1. + +--- + +## What Is Already Done (for reference) + +| Plan Task | Status | +|-----------|--------| +| D1 — Subscription provisioning | Done | +| D1 — Resource group structure (6 RGs) | Done | +| D1–D2 — VNet design & deployment (2 VNets, 4 subnets) | Done | +| D3 — NAT Gateway + static public IP | Done | +| D3–D4 — NSG rules (full inbound/outbound rule set) | Done | +| D2–D3 — Private DNS zones (4 zones, VNet links) | Done | +| D5 — Terraform state backend (`staccmdptfstate`) | Done | +| D1–D2 — VNet peerings (main ↔ transit, conditional hub) | Done | + +--- + +## Step 1 — ADLS Gen2 Storage Account (Plan: D4) + +**Module:** `modules/storage/` +**Owner:** Cloud Infra + +### 1.1 Implement `modules/storage/variables.tf` + +| Variable | Type | Description | +|----------|------|-------------| +| `location` | `string` | Azure region (`canadacentral`) | +| `environment` | `string` | Environment name (`prod`) | +| `project` | `string` | Project prefix (`mdp`) | +| `resource_group_name` | `string` | From `module.networking.rg_storage_name` | +| `subnet_pe_id` | `string` | From `module.networking.subnet_private_endpoints_id` | +| `dns_zone_dfs_id` | `string` | From `module.networking.dns_zone_dfs_id` | +| `replication_type` | `string` | `GRS` for prod | +| `tags` | `map(string)` | Standard tags | + +### 1.2 Implement `modules/storage/main.tf` + +Resources to create: + +1. **`azurerm_storage_account`** — `stglobalmdp` (e.g., `stglobalprodmdp`) + - `account_tier` = `Standard` + - `account_replication_type` = `var.replication_type` (GRS) + - `is_hns_enabled` = `true` (Data Lake Gen2) + - `public_network_access_enabled` = `false` + - `min_tls_version` = `TLS1_2` + - `allow_nested_items_to_be_public` = `false` + - `shared_access_key_enabled` = `false` (force Entra ID auth) + - `blob_properties` block: + - `delete_retention_policy` { `days = 7` } + - `container_delete_retention_policy` { `days = 7` } + +2. **`azurerm_storage_container`** — one for each container (6 total): + - `landing` — raw file drops from source systems + - `bronze` — ingested data (raw catalog external location) + - `silver` — cleansed, conformed data (curated catalog) + - `gold` — business-ready data (analytics catalog) + - `archive` — long-term retention (immutability policy) + - `checkpoints` — Structured Streaming / DLT checkpoints + +3. **`azurerm_storage_management_policy`** — immutability policy on `archive` container + - Rule: blobs in `archive` container → immutable for 365 days (or as defined by compliance) + +4. **`azurerm_private_endpoint`** — Private Endpoint for ADLS (DFS sub-resource) + - `name` = `pe--adls-dfs` + - `subnet_id` = `var.subnet_pe_id` + - `private_service_connection` → sub-resource type `dfs` + - `private_dns_zone_group` → link to `var.dns_zone_dfs_id` + +5. **`azurerm_private_endpoint`** — Private Endpoint for ADLS (Blob sub-resource) + - `name` = `pe--adls-blob` + - `subnet_id` = `var.subnet_pe_id` + - `private_service_connection` → sub-resource type `blob` + - Note: Add a `privatelink.blob.core.windows.net` DNS zone in the networking module if blob access is needed. Otherwise, DFS endpoint alone is sufficient for Databricks ABFSS access. + +### 1.3 Implement `modules/storage/outputs.tf` + +| Output | Value | Consumed By | +|--------|-------|-------------| +| `adls_id` | Storage account resource ID | `identity` module (role assignment) | +| `adls_name` | Storage account name | `unity-catalog` module (external locations) | +| `adls_primary_dfs_endpoint` | DFS endpoint URL | Validation / documentation | +| `container_names` | List of container names | Reference | + +### 1.4 Wire into root `main.tf` + +Uncomment the `module "storage"` block in `environments/prod/main.tf` (lines 36–47). All variable wiring is already in place. + +### 1.5 Validate + +```bash +cd Implementation/Terraform/environments/prod +terraform plan -target=module.storage +``` + +Verify: 1 storage account, 6 containers, 1 management policy, 1–2 private endpoints. + +--- + +## Step 2 — Key Vault (Plan: D4) + +**Module:** `modules/keyvault/` +**Owner:** Cloud Infra + +### 2.1 Implement `modules/keyvault/variables.tf` + +| Variable | Type | Description | +|----------|------|-------------| +| `location` | `string` | Azure region | +| `environment` | `string` | Environment name | +| `project` | `string` | Project prefix | +| `resource_group_name` | `string` | From `module.networking.rg_keyvault_name` | +| `subnet_pe_id` | `string` | From `module.networking.subnet_private_endpoints_id` | +| `dns_zone_vault_id` | `string` | From `module.networking.dns_zone_vault_id` | +| `tenant_id` | `string` | Azure AD tenant ID (add to `variables.tf` and `terraform.tfvars` at root level) | +| `tags` | `map(string)` | Standard tags | + +### 2.2 Implement `modules/keyvault/main.tf` + +Resources to create: + +1. **`azurerm_key_vault`** — `kv--mdp` (e.g., `kv-mdp-prod`) + - `sku_name` = `standard` + - `tenant_id` = `var.tenant_id` + - `soft_delete_retention_days` = `90` + - `purge_protection_enabled` = `true` + - `enable_rbac_authorization` = `true` (use Azure RBAC, not access policies) + - `public_network_access_enabled` = `false` + - `network_acls` block: + - `bypass` = `AzureServices` + - `default_action` = `Deny` + +2. **`azurerm_private_endpoint`** — Private Endpoint for Key Vault + - `name` = `pe--kv` + - `subnet_id` = `var.subnet_pe_id` + - `private_service_connection` → sub-resource type `vault` + - `private_dns_zone_group` → link to `var.dns_zone_vault_id` + +3. **`azurerm_key_vault_secret`** (x2) — placeholder secrets + - `databricks-pat` = `"PLACEHOLDER"` (to be replaced after workspace deployment) + - `jdbc-source-placeholder` = `"PLACEHOLDER"` (to be replaced with actual credentials) + - Note: These use RBAC, so the deploying principal needs `Key Vault Secrets Officer` role. + +### 2.3 Implement `modules/keyvault/outputs.tf` + +| Output | Value | Consumed By | +|--------|-------|-------------| +| `keyvault_id` | Key Vault resource ID | `databricks-workspace` module (secret scope), `identity` module | +| `keyvault_name` | Key Vault name | Reference | +| `keyvault_uri` | Key Vault URI | Databricks secret scope configuration | + +### 2.4 Root-level changes + +1. Add `tenant_id` variable to `environments/prod/variables.tf`: + ```hcl + variable "tenant_id" { + description = "Azure AD tenant ID" + type = string + } + ``` +2. Add `tenant_id` value to `environments/prod/terraform.tfvars`. +3. Pass `tenant_id` into the `module "keyvault"` block and uncomment it (lines 54–64). + +### 2.5 Validate + +```bash +terraform plan -target=module.keyvault +``` + +Verify: 1 Key Vault, 1 private endpoint, 2 placeholder secrets. + +--- + +## Step 3 — Monitoring Foundation (Plan: D4–D5) + +**Module:** `modules/monitoring/` +**Owner:** Cloud Infra + +### 3.1 Implement `modules/monitoring/variables.tf` + +| Variable | Type | Description | +|----------|------|-------------| +| `location` | `string` | Azure region | +| `environment` | `string` | Environment name | +| `project` | `string` | Project prefix | +| `resource_group_name` | `string` | From `module.networking.rg_monitoring_name` | +| `tags` | `map(string)` | Standard tags | + +### 3.2 Implement `modules/monitoring/main.tf` + +Resources to create: + +1. **`azurerm_log_analytics_workspace`** — `law--mdp` + - `sku` = `PerGB2018` + - `retention_in_days` = `90` (align with Greenfield retention policy) + - `daily_quota_gb` = `-1` (unlimited initially; tune after baseline) + +2. **`azurerm_monitor_diagnostic_setting`** — for Key Vault + - `target_resource_id` = Key Vault ID (passed as variable; wire after Key Vault module is deployed) + - `log_analytics_workspace_id` = Log Analytics workspace ID + - Enabled log categories: `AuditEvent`, `AzurePolicyEvaluationDetails` + - Enabled metric category: `AllMetrics` + + > Note: Diagnostic settings for ADLS and Databricks will be added in later steps as those modules are deployed. Start with Key Vault only. + +### 3.3 Implement `modules/monitoring/outputs.tf` + +| Output | Value | Consumed By | +|--------|-------|-------------| +| `log_analytics_workspace_id` | LAW resource ID | Databricks diagnostic settings (Week 2), ADLS diagnostic settings | +| `log_analytics_workspace_name` | LAW name | Reference | + +### 3.4 Wire into root `main.tf` + +Uncomment `module "monitoring"` block (lines 140–148). Add a `keyvault_id` input variable to the module to enable the Key Vault diagnostic setting. + +### 3.5 Validate + +```bash +terraform plan -target=module.monitoring +``` + +Verify: 1 Log Analytics workspace, 1 diagnostic setting (Key Vault). + +--- + +## Step 4 — Identity Module Foundation (Plan: D4–D5) + +**Module:** `modules/identity/` +**Owner:** Cloud Infra + +This step creates the managed identity and role assignments needed for Databricks to access ADLS and Key Vault. The full identity module will be extended in Week 2 for the Databricks access connector, but the ADLS role assignments should be in place now. + +### 4.1 Implement `modules/identity/variables.tf` + +| Variable | Type | Description | +|----------|------|-------------| +| `location` | `string` | Azure region | +| `environment` | `string` | Environment name | +| `project` | `string` | Project prefix | +| `rg_databricks_name` | `string` | Resource group for identity resources | +| `rg_storage_name` | `string` | Resource group for storage (not used directly, but for reference) | +| `rg_governance_name` | `string` | Resource group for governance | +| `storage_account_id` | `string` | ADLS Gen2 resource ID (for role assignment) | +| `tags` | `map(string)` | Standard tags | + +### 4.2 Implement `modules/identity/main.tf` + +Resources to create: + +1. **`azurerm_user_assigned_identity`** — `id--dbx-access` + - This is the managed identity that the Databricks access connector will use. + - Location: `var.location` + - Resource group: `var.rg_databricks_name` + +2. **`azurerm_databricks_access_connector`** — `dbxac-` + - `identity` block → type `UserAssigned`, identity IDs = [managed identity ID] + - Resource group: `var.rg_databricks_name` + +3. **`azurerm_role_assignment`** — Storage Blob Data Contributor + - `scope` = `var.storage_account_id` + - `role_definition_name` = `Storage Blob Data Contributor` + - `principal_id` = managed identity principal ID + +### 4.3 Implement `modules/identity/outputs.tf` + +| Output | Value | Consumed By | +|--------|-------|-------------| +| `managed_identity_id` | User-assigned identity resource ID | `databricks-workspace` module, `unity-catalog` module | +| `managed_identity_principal_id` | Identity principal ID | Reference | +| `access_connector_id` | Access connector resource ID | `unity-catalog` module (storage credential) | +| `access_connector_name` | Access connector name | Reference | + +### 4.4 Wire into root `main.tf` + +Uncomment `module "identity"` block (lines 71–82). Dependency on `module.storage.adls_id` is already wired. + +### 4.5 Validate + +```bash +terraform plan -target=module.identity +``` + +Verify: 1 managed identity, 1 access connector, 1 role assignment. + +--- + +## Step 5 — CI/CD Pipeline (Plan: D5) + +**Module:** `ci/` directory +**Owner:** DevOps + +### 5.1 Create `ci/azure-pipelines.yml` (or `.github/workflows/terraform.yml`) + +Pipeline stages: + +#### Stage 1 — Validate (runs on every PR) + +``` +trigger: none +pr: + branches: + include: [main] + +steps: + - terraform init -backend=false + - terraform validate + - terraform fmt -check -recursive + - tflint --init && tflint + - checkov -d . --framework terraform --soft-fail +``` + +#### Stage 2 — Plan (runs on every PR) + +``` +steps: + - terraform init (with backend config) + - terraform plan -out=tfplan + - Post plan output as PR comment (using az devops / gh CLI) +``` + +#### Stage 3 — Apply (runs on merge to `main` only) + +``` +trigger: + branches: + include: [main] + +steps: + - terraform init + - terraform plan -out=tfplan + - terraform apply tfplan +``` + +### 5.2 Create `ci/scripts/validate.sh` + +```bash +#!/usr/bin/env bash +set -euo pipefail + +echo "=== terraform fmt ===" +terraform fmt -check -recursive + +echo "=== terraform validate ===" +terraform init -backend=false +terraform validate + +echo "=== tflint ===" +tflint --init +tflint + +echo "=== checkov ===" +checkov -d . --framework terraform --soft-fail +``` + +### 5.3 Branch protection rules + +Configure on the Git repository (Azure DevOps or GitHub): +- Require PR for merges to `main` +- Require at least 1 approval +- Require `terraform plan` status check to pass +- Require `validate` status check to pass +- No direct pushes to `main` + +### 5.4 Service connection / secrets + +- Create a service principal or managed identity for Terraform CI/CD +- Grant it `Contributor` + `User Access Administrator` on the MDP subscription +- Store credentials as pipeline secrets (not in code) +- Grant `Storage Blob Data Contributor` on the tfstate storage account + +--- + +## Step 6 — Security Review Checkpoint (Plan: D5) + +**Owner:** Security Architect + +### 6.1 Review checklist + +The Security Architect must review and sign off on the following before proceeding to Week 2 (Databricks deployment): + +| # | Review Item | Evidence | +|---|-------------|----------| +| 1 | VNet topology matches approved design | `terraform state show` for VNets, subnets | +| 2 | NSG rules — no permissive inbound, Internet outbound denied | NSG rule export, compare against §3 of Phase 0 plan | +| 3 | Private Endpoints — ADLS and Key Vault accessible only via PE | `nslookup` from within VNet resolves to private IP | +| 4 | ADLS — public access disabled, HNS enabled, soft delete on | Storage account properties | +| 5 | Key Vault — purge protection on, public access disabled, RBAC auth | Key Vault properties | +| 6 | NAT Gateway — stable egress IP documented | Public IP address recorded for source system allowlisting | +| 7 | Terraform state — stored in separate subscription, blob versioning on | State storage account config | +| 8 | No secrets in code | `git log` search for keys/passwords, `checkov` scan results | +| 9 | Tags applied consistently | `az resource list --tag managed-by=terraform` | +| 10 | RBAC assignments — least privilege | Role assignments export | + +### 6.2 Sign-off artifact + +Produce a signed document: `Phase0_Week1_Security_Signoff.md` with findings, risk acceptance (if any), and approval to proceed to Databricks workspace deployment. + +--- + +## Execution Order and Dependencies + +``` +Step 1 (Storage) ──────┐ + ├──→ Step 4 (Identity) ──→ Step 6 (Security Review) +Step 2 (Key Vault) ─┬──┘ ↑ + │ │ + └──→ Step 3 (Monitoring) ──────────┘ + ↑ +Step 5 (CI/CD) ─────────────────────────────────────────┘ +``` + +- **Steps 1 & 2** can be implemented in parallel (no dependency on each other). +- **Step 3** (Monitoring) depends on Step 2 (Key Vault ID needed for diagnostic setting). +- **Step 4** (Identity) depends on Step 1 (ADLS storage account ID needed for role assignment). +- **Step 5** (CI/CD) can be done in parallel with Steps 1–4. +- **Step 6** (Security Review) is the gate — requires all preceding steps to be complete. + +--- + +## New Root Variables Required + +| Variable | Value | Added In | +|----------|-------|----------| +| `tenant_id` | Greenfield Azure AD tenant ID | Step 2 (Key Vault) | + +## terraform.tfvars Additions + +```hcl +tenant_id = "REPLACE-WITH-TENANT-ID" +```